HMVMXR0.cpp@ 45965

Last change on this file since 45965 was 45965, checked in by vboxsync, 12 years ago
VMM: Facility for getting the highest-priority pending interrupt from the APIC device.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.1 KB

Line
1	/* $Id: HMVMXR0.cpp 45965 2013-05-09 15:32:14Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147	/** Profiling macro. */
148	#ifdef HM_PROFILE_EXIT_DISPATCH
149	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
150	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
151	#else
152	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
153	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
154	#endif
155
156
157	/*******************************************************************************
158	* Structures and Typedefs *
159	*******************************************************************************/
160	/**
161	* A state structure for holding miscellaneous information across
162	* VMX non-root operation and restored after the transition.
163	*/
164	typedef struct VMXTRANSIENT
165	{
166	/** The host's rflags/eflags. */
167	RTCCUINTREG uEFlags;
168	#if HC_ARCH_BITS == 32
169	uint32_t u32Alignment0;
170	#endif
171	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
172	uint64_t u64LStarMsr;
173	/** The guest's TPR value used for TPR shadowing. */
174	uint8_t u8GuestTpr;
175	/** Alignment. */
176	uint8_t abAlignment0[6];
177
178	/** The basic VM-exit reason. */
179	uint16_t uExitReason;
180	/** Alignment. */
181	uint16_t u16Alignment0;
182	/** The VM-exit interruption error code. */
183	uint32_t uExitIntrErrorCode;
184	/** The VM-exit exit qualification. */
185	RTGCUINTPTR uExitQualification;
186	#if GC_ARCH_BITS == 32
187	/** Alignment. */
188	uint32_t u32Alignment1;
189	#endif
190
191	/** The VM-exit interruption-information field. */
192	uint32_t uExitIntrInfo;
193	/** The VM-exit instruction-length field. */
194	uint32_t cbInstr;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Alignment. */
198	uint8_t abAlignment1[5];
199
200	/** The VM-entry interruption-information field. */
201	uint32_t uEntryIntrInfo;
202	/** The VM-entry exception error code field. */
203	uint32_t uEntryXcptErrorCode;
204	/** The VM-entry instruction length field. */
205	uint32_t cbEntryInstr;
206
207	/** IDT-vectoring information field. */
208	uint32_t uIdtVectoringInfo;
209	/** IDT-vectoring error code. */
210	uint32_t uIdtVectoringErrorCode;
211
212	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
213	uint32_t fVmcsFieldsRead;
214	/** Whether TSC-offsetting should be setup before VM-entry. */
215	bool fUpdateTscOffsettingAndPreemptTimer;
216	/** Whether the VM-exit was caused by a page-fault during delivery of a
217	* contributary exception or a page-fault. */
218	bool fVectoringPF;
219	} VMXTRANSIENT, *PVMXTRANSIENT;
220	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
221	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
222	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
223
224
225	/**
226	* MSR-bitmap read permissions.
227	*/
228	typedef enum VMXMSREXITREAD
229	{
230	/** Reading this MSR causes a VM-exit. */
231	VMXMSREXIT_INTERCEPT_READ = 0xb,
232	/** Reading this MSR does not cause a VM-exit. */
233	VMXMSREXIT_PASSTHRU_READ
234	} VMXMSREXITREAD;
235
236	/**
237	* MSR-bitmap write permissions.
238	*/
239	typedef enum VMXMSREXITWRITE
240	{
241	/** Writing to this MSR causes a VM-exit. */
242	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
243	/** Writing to this MSR does not cause a VM-exit. */
244	VMXMSREXIT_PASSTHRU_WRITE
245	} VMXMSREXITWRITE;
246
247
248	/*******************************************************************************
249	* Internal Functions *
250	*******************************************************************************/
251	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
252	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
253	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
254	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
255	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
256	#endif
257	#ifndef HMVMX_USE_FUNCTION_TABLE
258	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
259	#define HMVMX_EXIT_DECL static int
260	#else
261	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
262	#endif
263
264	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308
309	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316
317
318	/*******************************************************************************
319	* Global Variables *
320	*******************************************************************************/
321	#ifdef HMVMX_USE_FUNCTION_TABLE
322	/**
323	* VM-exit handler.
324	*
325	* @returns VBox status code.
326	* @param pVCpu Pointer to the VMCPU.
327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
328	* out-of-sync. Make sure to update the required
329	* fields before using them.
330	* @param pVmxTransient Pointer to the VMX-transient structure.
331	*/
332	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333	/** Pointer to VM-exit handler. */
334	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
335
336	/**
337	* VMX_EXIT dispatch table.
338	*/
339	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
340	{
341	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
342	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
343	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
344	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
345	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
346	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
347	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
348	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
349	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
350	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
351	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
352	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
353	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
354	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
355	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
356	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
357	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
358	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
359	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
360	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
361	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
362	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
363	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
364	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
365	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
366	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
367	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
368	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
369	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
370	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
371	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
372	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
373	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
374	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
375	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
376	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
377	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
378	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
379	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
380	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
381	/* 40 UNDEFINED */ hmR0VmxExitPause,
382	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
383	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
384	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
385	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
386	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
387	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
388	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
389	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
390	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
391	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
392	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
393	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
394	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
395	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
396	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
397	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
398	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
399	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
400	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
401	};
402	#endif /* HMVMX_USE_FUNCTION_TABLE */
403
404	#ifdef VBOX_STRICT
405	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
406	{
407	/* 0 */ "(Not Used)",
408	/* 1 */ "VMCALL executed in VMX root operation.",
409	/* 2 */ "VMCLEAR with invalid physical address.",
410	/* 3 */ "VMCLEAR with VMXON pointer.",
411	/* 4 */ "VMLAUNCH with non-clear VMCS.",
412	/* 5 */ "VMRESUME with non-launched VMCS.",
413	/* 6 */ "VMRESUME after VMXOFF",
414	/* 7 */ "VM entry with invalid control fields.",
415	/* 8 */ "VM entry with invalid host state fields.",
416	/* 9 */ "VMPTRLD with invalid physical address.",
417	/* 10 */ "VMPTRLD with VMXON pointer.",
418	/* 11 */ "VMPTRLD with incorrect revision identifier.",
419	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
420	/* 13 */ "VMWRITE to read-only VMCS component.",
421	/* 14 */ "(Not Used)",
422	/* 15 */ "VMXON executed in VMX root operation.",
423	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
424	/* 17 */ "VM entry with non-launched executing VMCS.",
425	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
426	/* 19 */ "VMCALL with non-clear VMCS.",
427	/* 20 */ "VMCALL with invalid VM-exit control fields.",
428	/* 21 */ "(Not Used)",
429	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
430	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
431	/* 24 */ "VMCALL with invalid SMM-monitor features.",
432	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
433	/* 26 */ "VM entry with events blocked by MOV SS.",
434	/* 27 */ "(Not Used)",
435	/* 28 */ "Invalid operand to INVEPT/INVVPID."
436	};
437	#endif /* VBOX_STRICT */
438
439
440
441	/**
442	* Updates the VM's last error record. If there was a VMX instruction error,
443	* reads the error data from the VMCS and updates VCPU's last error record as
444	* well.
445	*
446	* @param pVM Pointer to the VM.
447	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
448	* VERR_VMX_UNABLE_TO_START_VM or
449	* VERR_VMX_INVALID_VMCS_FIELD).
450	* @param rc The error code.
451	*/
452	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
453	{
454	AssertPtr(pVM);
455	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
456	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
457	{
458	AssertPtrReturnVoid(pVCpu);
459	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
460	}
461	pVM->hm.s.lLastError = rc;
462	}
463
464
465	/**
466	* Reads the VM-entry interruption-information field from the VMCS into the VMX
467	* transient structure.
468	*
469	* @returns VBox status code.
470	* @param pVmxTransient Pointer to the VMX transient structure.
471	*
472	* @remarks No-long-jump zone!!!
473	*/
474	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
475	{
476	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
477	AssertRCReturn(rc, rc);
478	return VINF_SUCCESS;
479	}
480
481
482	/**
483	* Reads the VM-entry exception error code field from the VMCS into
484	* the VMX transient structure.
485	*
486	* @returns VBox status code.
487	* @param pVmxTransient Pointer to the VMX transient structure.
488	*
489	* @remarks No-long-jump zone!!!
490	*/
491	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
492	{
493	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
494	AssertRCReturn(rc, rc);
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* Reads the VM-entry exception error code field from the VMCS into
501	* the VMX transient structure.
502	*
503	* @returns VBox status code.
504	* @param pVCpu Pointer to the VMCPU.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-exit interruption-information field from the VMCS into the VMX
519	* transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
528	{
529	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the VM-exit interruption error code from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVCpu Pointer to the VMCPU.
543	* @param pVmxTransient Pointer to the VMX transient structure.
544	*/
545	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
546	{
547	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
548	{
549	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
550	AssertRCReturn(rc, rc);
551	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
552	}
553	return VINF_SUCCESS;
554	}
555
556
557	/**
558	* Reads the VM-exit instruction length field from the VMCS into the VMX
559	* transient structure.
560	*
561	* @returns VBox status code.
562	* @param pVCpu Pointer to the VMCPU.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Reads the exit qualification from the VMCS into the VMX transient structure.
579	*
580	* @returns VBox status code.
581	* @param pVCpu Pointer to the VMCPU.
582	* @param pVmxTransient Pointer to the VMX transient structure.
583	*/
584	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
585	{
586	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
587	{
588	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
589	AssertRCReturn(rc, rc);
590	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
591	}
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the IDT-vectoring information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*
603	* @remarks No-long-jump zone!!!
604	*/
605	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
606	{
607	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
608	{
609	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
610	AssertRCReturn(rc, rc);
611	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
612	}
613	return VINF_SUCCESS;
614	}
615
616
617	/**
618	* Reads the IDT-vectoring error code from the VMCS into the VMX
619	* transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*/
624	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
625	{
626	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
629	AssertRCReturn(rc, rc);
630	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
631	}
632	return VINF_SUCCESS;
633	}
634
635
636	/**
637	* Enters VMX root mode operation on the current CPU.
638	*
639	* @returns VBox status code.
640	* @param pVM Pointer to the VM (optional, can be NULL, after
641	* a resume).
642	* @param HCPhysCpuPage Physical address of the VMXON region.
643	* @param pvCpuPage Pointer to the VMXON region.
644	*/
645	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
646	{
647	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
648	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
649	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
650
651	if (pVM)
652	{
653	/* Write the VMCS revision dword to the VMXON region. */
654	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
655	}
656
657	/* Enable the VMX bit in CR4 if necessary. */
658	RTCCUINTREG uCr4 = ASMGetCR4();
659	if (!(uCr4 & X86_CR4_VMXE))
660	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
661
662	/* Enter VMX root mode. */
663	int rc = VMXEnable(HCPhysCpuPage);
664	if (RT_FAILURE(rc))
665	ASMSetCR4(uCr4);
666
667	return rc;
668	}
669
670
671	/**
672	* Exits VMX root mode operation on the current CPU.
673	*
674	* @returns VBox status code.
675	*/
676	static int hmR0VmxLeaveRootMode(void)
677	{
678	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
679
680	/* If we're for some reason not in VMX root mode, then don't leave it. */
681	if (ASMGetCR4() & X86_CR4_VMXE)
682	{
683	/* Exit VMX root mode and clear the VMX bit in CR4 */
684	VMXDisable();
685	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
686	return VINF_SUCCESS;
687	}
688
689	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
690	}
691
692
693	/**
694	* Allocates and maps one physically contiguous page. The allocated page is
695	* zero'd out. (Used by various VT-x structures).
696	*
697	* @returns IPRT status code.
698	* @param pMemObj Pointer to the ring-0 memory object.
699	* @param ppVirt Where to store the virtual address of the
700	* allocation.
701	* @param pPhys Where to store the physical address of the
702	* allocation.
703	*/
704	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
705	{
706	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
707	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
708	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
709
710	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
711	if (RT_FAILURE(rc))
712	return rc;
713	ppVirt = RTR0MemObjAddress(pMemObj);
714	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
715	ASMMemZero32(*ppVirt, PAGE_SIZE);
716	return VINF_SUCCESS;
717	}
718
719
720	/**
721	* Frees and unmaps an allocated physical page.
722	*
723	* @param pMemObj Pointer to the ring-0 memory object.
724	* @param ppVirt Where to re-initialize the virtual address of
725	* allocation as 0.
726	* @param pHCPhys Where to re-initialize the physical address of the
727	* allocation as 0.
728	*/
729	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
730	{
731	AssertPtr(pMemObj);
732	AssertPtr(ppVirt);
733	AssertPtr(pHCPhys);
734	if (*pMemObj != NIL_RTR0MEMOBJ)
735	{
736	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
737	AssertRC(rc);
738	*pMemObj = NIL_RTR0MEMOBJ;
739	*ppVirt = 0;
740	*pHCPhys = 0;
741	}
742	}
743
744
745	/**
746	* Worker function to free VT-x related structures.
747	*
748	* @returns IPRT status code.
749	* @param pVM Pointer to the VM.
750	*/
751	static void hmR0VmxStructsFree(PVM pVM)
752	{
753	for (VMCPUID i = 0; i < pVM->cCpus; i++)
754	{
755	PVMCPU pVCpu = &pVM->aCpus[i];
756	AssertPtr(pVCpu);
757
758	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
759	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
761	#endif
762
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
765
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
768	}
769
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
772	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
773	#endif
774	}
775
776
777	/**
778	* Worker function to allocate VT-x related VM structures.
779	*
780	* @returns IPRT status code.
781	* @param pVM Pointer to the VM.
782	*/
783	static int hmR0VmxStructsAlloc(PVM pVM)
784	{
785	/*
786	* Initialize members up-front so we can cleanup properly on allocation failure.
787	*/
788	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
789	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
790	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
791	pVM->hm.s.vmx.HCPhys##a_Name = 0;
792
793	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
794	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
795	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
796	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
797
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
800	#endif
801	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
802
803	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
804	for (VMCPUID i = 0; i < pVM->cCpus; i++)
805	{
806	PVMCPU pVCpu = &pVM->aCpus[i];
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
813	#endif
814	}
815	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
816	#undef VMXLOCAL_INIT_VM_MEMOBJ
817
818	/*
819	* Allocate all the VT-x structures.
820	*/
821	int rc = VINF_SUCCESS;
822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
823	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
827	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
828	#endif
829
830	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
831	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
832	{
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
834	&pVM->hm.s.vmx.HCPhysApicAccess);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	/*
840	* Initialize per-VCPU VT-x structures.
841	*/
842	for (VMCPUID i =0; i < pVM->cCpus; i++)
843	{
844	PVMCPU pVCpu = &pVM->aCpus[i];
845	AssertPtr(pVCpu);
846
847	/* Allocate the VM control structure (VMCS). */
848	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
854	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
855	{
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
857	&pVCpu->hm.s.vmx.HCPhysVirtApic);
858	if (RT_FAILURE(rc))
859	goto cleanup;
860	}
861
862	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
864	{
865	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
866	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867	if (RT_FAILURE(rc))
868	goto cleanup;
869	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
870	}
871
872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
873	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
874	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875	if (RT_FAILURE(rc))
876	goto cleanup;
877
878	/* Allocate the VM-exit MSR-load page for the host MSRs. */
879	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	#endif
883	}
884
885	return VINF_SUCCESS;
886
887	cleanup:
888	hmR0VmxStructsFree(pVM);
889	return rc;
890	}
891
892
893	/**
894	* Does global VT-x initialization (called during module initialization).
895	*
896	* @returns VBox status code.
897	*/
898	VMMR0DECL(int) VMXR0GlobalInit(void)
899	{
900	#ifdef HMVMX_USE_FUNCTION_TABLE
901	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
902	# ifdef VBOX_STRICT
903	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
904	Assert(g_apfnVMExitHandlers[i]);
905	# endif
906	#endif
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Does global VT-x termination (called during module termination).
913	*/
914	VMMR0DECL(void) VMXR0GlobalTerm()
915	{
916	/* Nothing to do currently. */
917	}
918
919
920	/**
921	* Sets up and activates VT-x on the current CPU.
922	*
923	* @returns VBox status code.
924	* @param pCpu Pointer to the global CPU info struct.
925	* @param pVM Pointer to the VM (can be NULL after a host resume
926	* operation).
927	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
928	* fEnabledByHost is true).
929	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
930	* @a fEnabledByHost is true).
931	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
932	* enable VT-x/AMD-V on the host.
933	*/
934	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
935	{
936	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
937	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
938
939	if (!fEnabledByHost)
940	{
941	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
942	if (RT_FAILURE(rc))
943	return rc;
944	}
945
946	/*
947	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
948	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
949	* each time while reusing a VPID after hitting the MaxASID limit once.
950	*/
951	if ( pVM
952	&& pVM->hm.s.vmx.fVpid
953	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
954	{
955	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
956	pCpu->fFlushAsidBeforeUse = false;
957	}
958	else
959	pCpu->fFlushAsidBeforeUse = true;
960
961	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
962	++pCpu->cTlbFlushes;
963
964	return VINF_SUCCESS;
965	}
966
967
968	/**
969	* Deactivates VT-x on the current CPU.
970	*
971	* @returns VBox status code.
972	* @param pCpu Pointer to the global CPU info struct.
973	* @param pvCpuPage Pointer to the VMXON region.
974	* @param HCPhysCpuPage Physical address of the VMXON region.
975	*/
976	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
977	{
978	NOREF(pCpu);
979	NOREF(pvCpuPage);
980	NOREF(HCPhysCpuPage);
981
982	return hmR0VmxLeaveRootMode();
983	}
984
985
986	/**
987	* Sets the permission bits for the specified MSR in the MSR bitmap.
988	*
989	* @param pVCpu Pointer to the VMCPU.
990	* @param uMSR The MSR value.
991	* @param enmRead Whether reading this MSR causes a VM-exit.
992	* @param enmWrite Whether writing this MSR causes a VM-exit.
993	*/
994	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
995	{
996	int32_t iBit;
997	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
998
999	/*
1000	* Layout:
1001	* 0x000 - 0x3ff - Low MSR read bits
1002	* 0x400 - 0x7ff - High MSR read bits
1003	* 0x800 - 0xbff - Low MSR write bits
1004	* 0xc00 - 0xfff - High MSR write bits
1005	*/
1006	if (uMsr <= 0x00001FFF)
1007	iBit = uMsr;
1008	else if ( uMsr >= 0xC0000000
1009	&& uMsr <= 0xC0001FFF)
1010	{
1011	iBit = (uMsr - 0xC0000000);
1012	pbMsrBitmap += 0x400;
1013	}
1014	else
1015	{
1016	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1017	return;
1018	}
1019
1020	Assert(iBit <= 0x1fff);
1021	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1022	ASMBitSet(pbMsrBitmap, iBit);
1023	else
1024	ASMBitClear(pbMsrBitmap, iBit);
1025
1026	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1027	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1028	else
1029	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1030	}
1031
1032
1033	/**
1034	* Flushes the TLB using EPT.
1035	*
1036	* @returns VBox status code.
1037	* @param pVM Pointer to the VM.
1038	* @param pVCpu Pointer to the VMCPU.
1039	* @param enmFlush Type of flush.
1040	*/
1041	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1042	{
1043	AssertPtr(pVM);
1044	Assert(pVM->hm.s.fNestedPaging);
1045
1046	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1047
1048	uint64_t descriptor[2];
1049	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1050	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1051
1052	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1053	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1054	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1055	}
1056
1057
1058	/**
1059	* Flushes the TLB using VPID.
1060	*
1061	* @returns VBox status code.
1062	* @param pVM Pointer to the VM.
1063	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1064	* enmFlush).
1065	* @param enmFlush Type of flush.
1066	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1067	* on @a enmFlush).
1068	*/
1069	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1070	{
1071	AssertPtr(pVM);
1072	Assert(pVM->hm.s.vmx.fVpid);
1073
1074	uint64_t descriptor[2];
1075	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1076	{
1077	descriptor[0] = 0;
1078	descriptor[1] = 0;
1079	}
1080	else
1081	{
1082	AssertPtr(pVCpu);
1083	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1084	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1085	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1086	descriptor[1] = GCPtr;
1087	}
1088
1089	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1090	AssertMsg(rc == VINF_SUCCESS,
1091	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1092	if ( RT_SUCCESS(rc)
1093	&& pVCpu)
1094	{
1095	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1096	}
1097	}
1098
1099
1100	/**
1101	* Invalidates a guest page by guest virtual address. Only relevant for
1102	* EPT/VPID, otherwise there is nothing really to invalidate.
1103	*
1104	* @returns VBox status code.
1105	* @param pVM Pointer to the VM.
1106	* @param pVCpu Pointer to the VMCPU.
1107	* @param GCVirt Guest virtual address of the page to invalidate.
1108	*/
1109	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1110	{
1111	AssertPtr(pVM);
1112	AssertPtr(pVCpu);
1113	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1114
1115	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1116	if (!fFlushPending)
1117	{
1118	/*
1119	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1120	* See @bugref{6043} and @bugref{6177}.
1121	*
1122	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1123	* function maybe called in a loop with individual addresses.
1124	*/
1125	if (pVM->hm.s.vmx.fVpid)
1126	{
1127	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1128	{
1129	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1130	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1131	}
1132	else
1133	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1134	}
1135	else if (pVM->hm.s.fNestedPaging)
1136	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1137	}
1138
1139	return VINF_SUCCESS;
1140	}
1141
1142
1143	/**
1144	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1145	* otherwise there is nothing really to invalidate.
1146	*
1147	* @returns VBox status code.
1148	* @param pVM Pointer to the VM.
1149	* @param pVCpu Pointer to the VMCPU.
1150	* @param GCPhys Guest physical address of the page to invalidate.
1151	*/
1152	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1153	{
1154	LogFlowFunc(("%RGp\n", GCPhys));
1155
1156	/*
1157	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1158	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1159	* This function might be called in a loop.
1160	*/
1161	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1163	return VINF_SUCCESS;
1164	}
1165
1166
1167	/**
1168	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1169	* case where neither EPT nor VPID is supported by the CPU.
1170	*
1171	* @param pVM Pointer to the VM.
1172	* @param pVCpu Pointer to the VMCPU.
1173	*
1174	* @remarks Called with interrupts disabled.
1175	*/
1176	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1177	{
1178	NOREF(pVM);
1179	AssertPtr(pVCpu);
1180	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1181	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1182
1183	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1184	AssertPtr(pCpu);
1185
1186	pVCpu->hm.s.TlbShootdown.cPages = 0;
1187	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1188	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1189	pVCpu->hm.s.fForceTLBFlush = false;
1190	return;
1191	}
1192
1193
1194	/**
1195	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1196	*
1197	* @param pVM Pointer to the VM.
1198	* @param pVCpu Pointer to the VMCPU.
1199	* @remarks All references to "ASID" in this function pertains to "VPID" in
1200	* Intel's nomenclature. The reason is, to avoid confusion in compare
1201	* statements since the host-CPU copies are named "ASID".
1202	*
1203	* @remarks Called with interrupts disabled.
1204	*/
1205	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1206	{
1207	AssertPtr(pVM);
1208	AssertPtr(pVCpu);
1209	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1210	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1211	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1212
1213	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1214	AssertPtr(pCpu);
1215
1216	/*
1217	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1218	* This can happen both for start & resume due to long jumps back to ring-3.
1219	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1220	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1221	*/
1222	bool fNewASID = false;
1223	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1224	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1225	{
1226	pVCpu->hm.s.fForceTLBFlush = true;
1227	fNewASID = true;
1228	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1229	}
1230
1231	/*
1232	* Check for explicit TLB shootdowns.
1233	*/
1234	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1235	{
1236	pVCpu->hm.s.fForceTLBFlush = true;
1237	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1238	}
1239
1240	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1241	if (pVCpu->hm.s.fForceTLBFlush)
1242	{
1243	if (fNewASID)
1244	{
1245	++pCpu->uCurrentAsid;
1246	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1247	{
1248	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1249	pCpu->cTlbFlushes++;
1250	pCpu->fFlushAsidBeforeUse = true;
1251	}
1252
1253	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1254	if (pCpu->fFlushAsidBeforeUse)
1255	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1256	}
1257	else
1258	{
1259	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1260	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1261	else
1262	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1263	}
1264
1265	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1266	pVCpu->hm.s.fForceTLBFlush = false;
1267	}
1268	else
1269	{
1270	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1271	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1272	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1273	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1274
1275	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1276	* not be executed. See hmQueueInvlPage() where it is commented
1277	* out. Support individual entry flushing someday. */
1278	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1279	{
1280	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1281
1282	/*
1283	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1284	* as supported by the CPU.
1285	*/
1286	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1287	{
1288	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1289	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1290	}
1291	else
1292	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1293	}
1294	else
1295	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1296	}
1297	pVCpu->hm.s.TlbShootdown.cPages = 0;
1298	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1299
1300	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1301	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1302	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1304	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1305	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1306
1307	/* Update VMCS with the VPID. */
1308	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1309	AssertRC(rc);
1310	}
1311
1312
1313	/**
1314	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1315	*
1316	* @returns VBox status code.
1317	* @param pVM Pointer to the VM.
1318	* @param pVCpu Pointer to the VMCPU.
1319	*
1320	* @remarks Called with interrupts disabled.
1321	*/
1322	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1323	{
1324	AssertPtr(pVM);
1325	AssertPtr(pVCpu);
1326	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1327	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1328
1329	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1330	AssertPtr(pCpu);
1331
1332	/*
1333	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1334	* This can happen both for start & resume due to long jumps back to ring-3.
1335	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1336	*/
1337	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1338	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1339	{
1340	pVCpu->hm.s.fForceTLBFlush = true;
1341	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1342	}
1343
1344	/* Check for explicit TLB shootdown flushes. */
1345	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1346	{
1347	pVCpu->hm.s.fForceTLBFlush = true;
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	}
1350
1351	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1352	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1353
1354	if (pVCpu->hm.s.fForceTLBFlush)
1355	{
1356	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1357	pVCpu->hm.s.fForceTLBFlush = false;
1358	}
1359	else
1360	{
1361	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1362	* not be executed. See hmQueueInvlPage() where it is commented
1363	* out. Support individual entry flushing someday. */
1364	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1365	{
1366	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1368	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1369	}
1370	else
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1372	}
1373
1374	pVCpu->hm.s.TlbShootdown.cPages = 0;
1375	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1376	}
1377
1378
1379	/**
1380	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1381	*
1382	* @returns VBox status code.
1383	* @param pVM Pointer to the VM.
1384	* @param pVCpu Pointer to the VMCPU.
1385	*
1386	* @remarks Called with interrupts disabled.
1387	*/
1388	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1389	{
1390	AssertPtr(pVM);
1391	AssertPtr(pVCpu);
1392	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1393	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1394
1395	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1396
1397	/*
1398	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1399	* This can happen both for start & resume due to long jumps back to ring-3.
1400	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1401	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1402	*/
1403	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1404	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1405	{
1406	pVCpu->hm.s.fForceTLBFlush = true;
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1408	}
1409
1410	/* Check for explicit TLB shootdown flushes. */
1411	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1412	{
1413	/*
1414	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1415	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1416	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1417	*/
1418	pVCpu->hm.s.fForceTLBFlush = true;
1419	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1420	}
1421
1422	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1423	if (pVCpu->hm.s.fForceTLBFlush)
1424	{
1425	++pCpu->uCurrentAsid;
1426	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1427	{
1428	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1429	pCpu->fFlushAsidBeforeUse = true;
1430	pCpu->cTlbFlushes++;
1431	}
1432
1433	pVCpu->hm.s.fForceTLBFlush = false;
1434	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1435	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1436	if (pCpu->fFlushAsidBeforeUse)
1437	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1438	}
1439	else
1440	{
1441	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1442	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1443	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1444	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1445
1446	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1447	* not be executed. See hmQueueInvlPage() where it is commented
1448	* out. Support individual entry flushing someday. */
1449	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1450	{
1451	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1452	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1453	{
1454	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1455	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1456	}
1457	else
1458	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1459	}
1460	else
1461	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1462	}
1463
1464	pVCpu->hm.s.TlbShootdown.cPages = 0;
1465	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1466
1467	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1468	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1469	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1471	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1472	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1473
1474	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1475	AssertRC(rc);
1476	}
1477
1478
1479	/**
1480	* Flushes the guest TLB entry based on CPU capabilities.
1481	*
1482	* @param pVCpu Pointer to the VMCPU.
1483	*/
1484	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1485	{
1486	PVM pVM = pVCpu->CTX_SUFF(pVM);
1487	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1488	{
1489	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1491	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1492	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1493	default:
1494	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1495	break;
1496	}
1497	}
1498
1499
1500	/**
1501	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1502	* TLB entries from the host TLB before VM-entry.
1503	*
1504	* @returns VBox status code.
1505	* @param pVM Pointer to the VM.
1506	*/
1507	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1508	{
1509	/*
1510	* Determine optimal flush type for nested paging.
1511	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1512	* guest execution (see hmR3InitFinalizeR0()).
1513	*/
1514	if (pVM->hm.s.fNestedPaging)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1517	{
1518	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1520	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1521	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1522	else
1523	{
1524	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1525	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1526	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1527	}
1528
1529	/* Make sure the write-back cacheable memory type for EPT is supported. */
1530	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1531	{
1532	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1533	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1534	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1535	}
1536	}
1537	else
1538	{
1539	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1540	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1541	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1542	}
1543	}
1544
1545	/*
1546	* Determine optimal flush type for VPID.
1547	*/
1548	if (pVM->hm.s.vmx.fVpid)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1551	{
1552	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1554	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1555	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1556	else
1557	{
1558	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1561	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1562	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1563	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1564	pVM->hm.s.vmx.fVpid = false;
1565	}
1566	}
1567	else
1568	{
1569	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1570	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1571	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1572	pVM->hm.s.vmx.fVpid = false;
1573	}
1574	}
1575
1576	/*
1577	* Setup the handler for flushing tagged-TLBs.
1578	*/
1579	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1581	else if (pVM->hm.s.fNestedPaging)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1583	else if (pVM->hm.s.vmx.fVpid)
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1585	else
1586	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1587	return VINF_SUCCESS;
1588	}
1589
1590
1591	/**
1592	* Sets up pin-based VM-execution controls in the VMCS.
1593	*
1594	* @returns VBox status code.
1595	* @param pVM Pointer to the VM.
1596	* @param pVCpu Pointer to the VMCPU.
1597	*/
1598	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1599	{
1600	AssertPtr(pVM);
1601	AssertPtr(pVCpu);
1602
1603	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1604	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1605
1606	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1607	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1608	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1609
1610	/* Enable the VMX preemption timer. */
1611	if (pVM->hm.s.vmx.fUsePreemptTimer)
1612	{
1613	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1614	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1615	}
1616
1617	if ((val & zap) != val)
1618	{
1619	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1620	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1621	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1622	}
1623
1624	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1625	AssertRCReturn(rc, rc);
1626
1627	/* Update VCPU with the currently set pin-based VM-execution controls. */
1628	pVCpu->hm.s.vmx.u32PinCtls = val;
1629	return rc;
1630	}
1631
1632
1633	/**
1634	* Sets up processor-based VM-execution controls in the VMCS.
1635	*
1636	* @returns VBox status code.
1637	* @param pVM Pointer to the VM.
1638	* @param pVMCPU Pointer to the VMCPU.
1639	*/
1640	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1641	{
1642	AssertPtr(pVM);
1643	AssertPtr(pVCpu);
1644
1645	int rc = VERR_INTERNAL_ERROR_5;
1646	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1647	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1648
1649	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1654	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1655	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1656
1657	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1658	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1659	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1660	{
1661	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1662	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1663	}
1664
1665	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1666	if (!pVM->hm.s.fNestedPaging)
1667	{
1668	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1669	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1670	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1671	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1672	}
1673
1674	/* Use TPR shadowing if supported by the CPU. */
1675	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1676	{
1677	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1678	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1679	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1680	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1681	AssertRCReturn(rc, rc);
1682
1683	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1684	/* CR8 writes causes a VM-exit based on TPR threshold. */
1685	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1686	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1687	}
1688	else
1689	{
1690	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1691	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1692	}
1693
1694	/* Use MSR-bitmaps if supported by the CPU. */
1695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1696	{
1697	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1698
1699	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1701	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1702	AssertRCReturn(rc, rc);
1703
1704	/*
1705	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1706	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1707	*/
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1716	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1717	}
1718
1719	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1720	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1721	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1722
1723	if ((val & zap) != val)
1724	{
1725	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1726	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1727	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1728	}
1729
1730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1731	AssertRCReturn(rc, rc);
1732
1733	/* Update VCPU with the currently set processor-based VM-execution controls. */
1734	pVCpu->hm.s.vmx.u32ProcCtls = val;
1735
1736	/*
1737	* Secondary processor-based VM-execution controls.
1738	*/
1739	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1740	{
1741	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1742	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1743
1744	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1745	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1746
1747	if (pVM->hm.s.fNestedPaging)
1748	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1749	else
1750	{
1751	/*
1752	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1753	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1754	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1755	*/
1756	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1757	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1758	}
1759
1760	if (pVM->hm.s.vmx.fVpid)
1761	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1762
1763	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1764	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1765
1766	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1767	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1768	* done dynamically. */
1769	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1770	{
1771	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1772	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1773	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1774	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1775	AssertRCReturn(rc, rc);
1776	}
1777
1778	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1779	{
1780	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1781	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1782	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1783	}
1784
1785	if ((val & zap) != val)
1786	{
1787	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1788	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1789	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1790	}
1791
1792	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1793	AssertRCReturn(rc, rc);
1794
1795	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1796	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1797	}
1798
1799	return VINF_SUCCESS;
1800	}
1801
1802
1803	/**
1804	* Sets up miscellaneous (everything other than Pin & Processor-based
1805	* VM-execution) control fields in the VMCS.
1806	*
1807	* @returns VBox status code.
1808	* @param pVM Pointer to the VM.
1809	* @param pVCpu Pointer to the VMCPU.
1810	*/
1811	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1812	{
1813	AssertPtr(pVM);
1814	AssertPtr(pVCpu);
1815
1816	/** @todo Shouldn't we able to avoid initializing with 0? */
1817	int rc = VERR_GENERAL_FAILURE;
1818
1819	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1820	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1821	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1822
1823	/*
1824	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1825	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1826	* We thus use the exception bitmap to control it rather than use both.
1827	*/
1828	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1829	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1830
1831	/** @todo Explore possibility of using IO-bitmaps. */
1832	/* All IO & IOIO instructions cause VM-exits. */
1833	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1834	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1835
1836	/* Initialize the MSR-bitmap area. */
1837	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1838	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1839	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1840
1841	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1842	/* Setup MSR autoloading/storing. */
1843	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1845	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1846	AssertRCReturn(rc, rc);
1847	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1848	AssertRCReturn(rc, rc);
1849
1850	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1851	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1852	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1853	AssertRCReturn(rc, rc);
1854	#endif
1855
1856	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1857	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1858	AssertRCReturn(rc, rc);
1859
1860	/* Setup debug controls */
1861	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1862	AssertRCReturn(rc, rc);
1863	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1864	AssertRCReturn(rc, rc);
1865	return rc;
1866	}
1867
1868
1869	/**
1870	* Sets up the initial exception bitmap in the VMCS based on static conditions
1871	* (i.e. conditions that cannot ever change at runtime).
1872	*
1873	* @returns VBox status code.
1874	* @param pVM Pointer to the VM.
1875	* @param pVCpu Pointer to the VMCPU.
1876	*/
1877	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1878	{
1879	AssertPtr(pVM);
1880	AssertPtr(pVCpu);
1881
1882	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1883
1884	uint32_t u32XcptBitmap = 0;
1885
1886	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1887	if (!pVM->hm.s.fNestedPaging)
1888	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1889
1890	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1891	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1892	AssertRCReturn(rc, rc);
1893	return rc;
1894	}
1895
1896
1897	/**
1898	* Sets up the initial guest-state mask. The guest-state mask is consulted
1899	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1900	* for the nested virtualization case (as it would cause a VM-exit).
1901	*
1902	* @param pVCpu Pointer to the VMCPU.
1903	*/
1904	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1905	{
1906	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1907	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1908	return VINF_SUCCESS;
1909	}
1910
1911
1912	/**
1913	* Does per-VM VT-x initialization.
1914	*
1915	* @returns VBox status code.
1916	* @param pVM Pointer to the VM.
1917	*/
1918	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1919	{
1920	LogFlowFunc(("pVM=%p\n", pVM));
1921
1922	int rc = hmR0VmxStructsAlloc(pVM);
1923	if (RT_FAILURE(rc))
1924	{
1925	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1926	return rc;
1927	}
1928
1929	return VINF_SUCCESS;
1930	}
1931
1932
1933	/**
1934	* Does per-VM VT-x termination.
1935	*
1936	* @returns VBox status code.
1937	* @param pVM Pointer to the VM.
1938	*/
1939	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1940	{
1941	LogFlowFunc(("pVM=%p\n", pVM));
1942
1943	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1944	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1945	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1946	#endif
1947	hmR0VmxStructsFree(pVM);
1948	return VINF_SUCCESS;
1949	}
1950
1951
1952	/**
1953	* Sets up the VM for execution under VT-x.
1954	* This function is only called once per-VM during initalization.
1955	*
1956	* @returns VBox status code.
1957	* @param pVM Pointer to the VM.
1958	*/
1959	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1960	{
1961	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1962	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1963
1964	LogFlowFunc(("pVM=%p\n", pVM));
1965
1966	/*
1967	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1968	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1969	*/
1970	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1971	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1972	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1973	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1974	{
1975	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1976	return VERR_INTERNAL_ERROR;
1977	}
1978
1979	/* Initialize these always, see hmR3InitFinalizeR0().*/
1980	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1981	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1982
1983	/* Setup the tagged-TLB flush handlers. */
1984	int rc = hmR0VmxSetupTaggedTlb(pVM);
1985	if (RT_FAILURE(rc))
1986	{
1987	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1988	return rc;
1989	}
1990
1991	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1992	{
1993	PVMCPU pVCpu = &pVM->aCpus[i];
1994	AssertPtr(pVCpu);
1995	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1996
1997	/* Set revision dword at the beginning of the VMCS structure. */
1998	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1999
2000	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2001	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2002	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2003	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2004
2005	/* Load this VMCS as the current VMCS. */
2006	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2007	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2008	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2009
2010	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2011	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2012	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2013
2014	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2015	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2016	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2017
2018	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2019	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2020	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2021
2022	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2023	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2024	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2025
2026	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2027	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2028	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2029
2030	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2031	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2032	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2033	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2034	#endif
2035
2036	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2037	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2038	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2040
2041	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2042	}
2043
2044	return VINF_SUCCESS;
2045	}
2046
2047
2048	/**
2049	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2050	* the VMCS.
2051	*
2052	* @returns VBox status code.
2053	* @param pVM Pointer to the VM.
2054	* @param pVCpu Pointer to the VMCPU.
2055	*/
2056	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2057	{
2058	RTCCUINTREG uReg = ASMGetCR0();
2059	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2060	AssertRCReturn(rc, rc);
2061
2062	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2063	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2064	if (HMVMX_IS_64BIT_HOST_MODE())
2065	{
2066	uint64_t uRegCR3 = HMR0Get64bitCR3();
2067	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2068	}
2069	else
2070	#endif
2071	{
2072	uReg = ASMGetCR3();
2073	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2074	}
2075	AssertRCReturn(rc, rc);
2076
2077	uReg = ASMGetCR4();
2078	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2079	AssertRCReturn(rc, rc);
2080	return rc;
2081	}
2082
2083
2084	/**
2085	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2086	* the host-state area in the VMCS.
2087	*
2088	* @returns VBox status code.
2089	* @param pVM Pointer to the VM.
2090	* @param pVCpu Pointer to the VMCPU.
2091	*/
2092	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2093	{
2094	int rc = VERR_INTERNAL_ERROR_5;
2095	RTSEL uSelCS = 0;
2096	RTSEL uSelSS = 0;
2097	RTSEL uSelDS = 0;
2098	RTSEL uSelES = 0;
2099	RTSEL uSelFS = 0;
2100	RTSEL uSelGS = 0;
2101	RTSEL uSelTR = 0;
2102
2103	/*
2104	* Host Selector registers.
2105	*/
2106	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2107	if (HMVMX_IS_64BIT_HOST_MODE())
2108	{
2109	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2110	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2111	}
2112	else
2113	{
2114	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2115	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2116	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2117	}
2118	#else
2119	uSelCS = ASMGetCS();
2120	uSelSS = ASMGetSS();
2121	#endif
2122
2123	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2124	uSelTR = ASMGetTR();
2125
2126	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2127	/** @todo Verify if we have any platform that actually run with DS or ES with
2128	* RPL != 0 in kernel space. */
2129	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2130	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2131	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2132	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2133	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2134	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2135	Assert(uSelCS != 0);
2136	Assert(uSelTR != 0);
2137
2138	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2139	#if 0
2140	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2141	Assert(uSelSS != 0);
2142	#endif
2143
2144	/* Write these host selector fields into the host-state area in the VMCS. */
2145	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2146	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2147	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2148	#if 0
2149	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2150	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2151	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2152	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2153	#endif
2154	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2155
2156	/*
2157	* Host GDTR and IDTR.
2158	*/
2159	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2160	* be safe to -not- save and restore GDTR and IDTR in the assembly
2161	* code and just do it here and don't care if the limits are zapped on
2162	* VM-exit. */
2163	RTGDTR Gdtr;
2164	RT_ZERO(Gdtr);
2165	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2166	if (HMVMX_IS_64BIT_HOST_MODE())
2167	{
2168	X86XDTR64 Gdtr64;
2169	X86XDTR64 Idtr64;
2170	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2171	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2172	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2173
2174	Gdtr.cbGdt = Gdtr64.cb;
2175	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2176	}
2177	else
2178	#endif
2179	{
2180	RTIDTR Idtr;
2181	ASMGetGDTR(&Gdtr);
2182	ASMGetIDTR(&Idtr);
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2184	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2185	}
2186
2187	/*
2188	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2189	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2190	*/
2191	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2192	{
2193	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2194	return VERR_VMX_INVALID_HOST_STATE;
2195	}
2196
2197	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2198	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2199	if (HMVMX_IS_64BIT_HOST_MODE())
2200	{
2201	/* We need the 64-bit TR base for hybrid darwin. */
2202	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2203	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2204	}
2205	else
2206	#endif
2207	{
2208	uintptr_t uTRBase;
2209	#if HC_ARCH_BITS == 64
2210	uTRBase = X86DESC64_BASE(pDesc);
2211	#else
2212	uTRBase = X86DESC_BASE(pDesc);
2213	#endif
2214	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2215	}
2216	AssertRCReturn(rc, rc);
2217
2218	/*
2219	* Host FS base and GS base.
2220	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2221	* would take care of the bases. In 64-bit, the MSRs come into play.
2222	*/
2223	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2224	if (HMVMX_IS_64BIT_HOST_MODE())
2225	{
2226	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2227	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2228	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2229	AssertRCReturn(rc, rc);
2230	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2231	AssertRCReturn(rc, rc);
2232	}
2233	#endif
2234	return rc;
2235	}
2236
2237
2238	/**
2239	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2240	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2241	* the host after every successful VM exit.
2242	*
2243	* @returns VBox status code.
2244	* @param pVM Pointer to the VM.
2245	* @param pVCpu Pointer to the VMCPU.
2246	*/
2247	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2248	{
2249	AssertPtr(pVCpu);
2250	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2251
2252	int rc = VINF_SUCCESS;
2253	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2254	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2255	uint32_t cHostMsrs = 0;
2256	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2257
2258	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2259	{
2260	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2261	pHostMsr->u32Reserved = 0;
2262	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2263	if (CPUMIsGuestInLongMode(pVCpu))
2264	{
2265	/* Must match the EFER value in our 64 bits switcher. */
2266	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2267	}
2268	else
2269	# endif
2270	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2271	pHostMsr++; cHostMsrs++;
2272	}
2273
2274	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2275	if (HMVMX_IS_64BIT_HOST_MODE())
2276	{
2277	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2278	pHostMsr->u32Reserved = 0;
2279	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2280	pHostMsr++; cHostMsrs++;
2281	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2282	pHostMsr->u32Reserved = 0;
2283	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2284	pHostMsr++; cHostMsrs++;
2285	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2286	pHostMsr->u32Reserved = 0;
2287	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2288	pHostMsr++; cHostMsrs++;
2289	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2290	pHostMsr->u32Reserved = 0;
2291	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2292	pHostMsr++; cHostMsrs++;
2293	}
2294	# endif
2295
2296	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2297	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2298	{
2299	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2300	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2301	}
2302
2303	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2304	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2305
2306	/*
2307	* Host Sysenter MSRs.
2308	*/
2309	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2310	AssertRCReturn(rc, rc);
2311	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2312	if (HMVMX_IS_64BIT_HOST_MODE())
2313	{
2314	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2315	AssertRCReturn(rc, rc);
2316	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2317	}
2318	else
2319	{
2320	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2321	AssertRCReturn(rc, rc);
2322	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2323	}
2324	# elif HC_ARCH_BITS == 32
2325	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2326	AssertRCReturn(rc, rc);
2327	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2328	# else
2329	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2330	AssertRCReturn(rc, rc);
2331	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2332	# endif
2333	AssertRCReturn(rc, rc);
2334
2335	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2336	* hmR0VmxSetupExitCtls() !! */
2337	return rc;
2338	}
2339
2340
2341	/**
2342	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2343	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2344	* controls".
2345	*
2346	* @returns VBox status code.
2347	* @param pVCpu Pointer to the VMCPU.
2348	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2349	* out-of-sync. Make sure to update the required fields
2350	* before using them.
2351	*
2352	* @remarks No-long-jump zone!!!
2353	*/
2354	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2355	{
2356	int rc = VINF_SUCCESS;
2357	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2358	{
2359	PVM pVM = pVCpu->CTX_SUFF(pVM);
2360	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2361	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2362
2363	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2364	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2365
2366	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2367	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2368	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2369	else
2370	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2371
2372	/*
2373	* The following should not be set (since we're not in SMM mode):
2374	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2375	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2376	*/
2377
2378	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2379	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2380	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2381
2382	if ((val & zap) != val)
2383	{
2384	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2385	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2386	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2387	}
2388
2389	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2390	AssertRCReturn(rc, rc);
2391
2392	/* Update VCPU with the currently set VM-exit controls. */
2393	pVCpu->hm.s.vmx.u32EntryCtls = val;
2394	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2395	}
2396	return rc;
2397	}
2398
2399
2400	/**
2401	* Sets up the VM-exit controls in the VMCS.
2402	*
2403	* @returns VBox status code.
2404	* @param pVM Pointer to the VM.
2405	* @param pVCpu Pointer to the VMCPU.
2406	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2407	* out-of-sync. Make sure to update the required fields
2408	* before using them.
2409	*
2410	* @remarks requires EFER.
2411	*/
2412	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2413	{
2414	int rc = VINF_SUCCESS;
2415	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2416	{
2417	PVM pVM = pVCpu->CTX_SUFF(pVM);
2418	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2419	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2420
2421	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2422	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2423
2424	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2425	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2426	if (HMVMX_IS_64BIT_HOST_MODE())
2427	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2428	else
2429	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2430	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2431	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2432	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2433	else
2434	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2435	#endif
2436
2437	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2438	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2439
2440	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2441	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2442	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2443	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2444	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2445
2446	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2447	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2448
2449	if ((val & zap) != val)
2450	{
2451	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2452	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2453	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2454	}
2455
2456	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2457	AssertRCReturn(rc, rc);
2458
2459	/* Update VCPU with the currently set VM-exit controls. */
2460	pVCpu->hm.s.vmx.u32ExitCtls = val;
2461	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2462	}
2463	return rc;
2464	}
2465
2466
2467	/**
2468	* Loads the guest APIC and related state.
2469	*
2470	* @returns VBox status code.
2471	* @param pVM Pointer to the VM.
2472	* @param pVCpu Pointer to the VMCPU.
2473	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2474	* out-of-sync. Make sure to update the required fields
2475	* before using them.
2476	*/
2477	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2478	{
2479	int rc = VINF_SUCCESS;
2480	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2481	{
2482	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2483	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2484	{
2485	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2486
2487	bool fPendingIntr = false;
2488	uint8_t u8GuestTpr = 0;
2489	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr, NULL /* pu8PendingIntr */);
2490	AssertRCReturn(rc, rc);
2491
2492	/*
2493	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2494	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2495	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2496	* the interrupt when we VM-exit for other reasons.
2497	*/
2498	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2499	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2500	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2501	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2502
2503	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2504	AssertRCReturn(rc, rc);
2505
2506	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2507	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2508	{
2509	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2510	pMixedCtx->msrLSTAR = u8GuestTpr;
2511	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2512	{
2513	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2514	if (fPendingIntr)
2515	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2516	else
2517	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518	}
2519	}
2520	}
2521
2522	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2523	}
2524	return rc;
2525	}
2526
2527
2528	/**
2529	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2530	*
2531	* @returns
2532	* @param pVCpu Pointer to the VMCPU.
2533	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2534	* out-of-sync. Make sure to update the required fields
2535	* before using them.
2536	*
2537	* @remarks No-long-jump zone!!!
2538	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2539	*/
2540	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2541	{
2542	/*
2543	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2544	* inhibit interrupts or clear any existing interrupt-inhibition.
2545	*/
2546	uint32_t uIntrState = 0;
2547	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2548	{
2549	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2550	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2551	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2552	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2553	{
2554	/*
2555	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2556	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2557	*/
2558	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2559	}
2560	else if (pMixedCtx->eflags.Bits.u1IF)
2561	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2562	else
2563	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2564	}
2565	return uIntrState;
2566	}
2567
2568
2569	/**
2570	* Loads the guest's interruptibility-state into the guest-state area in the
2571	* VMCS.
2572	*
2573	* @returns VBox status code.
2574	* @param pVCpu Pointer to the VMCPU.
2575	* @param uIntrState The interruptibility-state to set.
2576	*/
2577	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2578	{
2579	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2580	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2581	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2582	AssertRCReturn(rc, rc);
2583	return rc;
2584	}
2585
2586
2587	/**
2588	* Loads the guest's RIP into the guest-state area in the VMCS.
2589	*
2590	* @returns VBox status code.
2591	* @param pVCpu Pointer to the VMCPU.
2592	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2593	* out-of-sync. Make sure to update the required fields
2594	* before using them.
2595	*
2596	* @remarks No-long-jump zone!!!
2597	*/
2598	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2599	{
2600	int rc = VINF_SUCCESS;
2601	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2602	{
2603	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2604	AssertRCReturn(rc, rc);
2605	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2606	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2607	}
2608	return rc;
2609	}
2610
2611
2612	/**
2613	* Loads the guest's RSP into the guest-state area in the VMCS.
2614	*
2615	* @returns VBox status code.
2616	* @param pVCpu Pointer to the VMCPU.
2617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2618	* out-of-sync. Make sure to update the required fields
2619	* before using them.
2620	*
2621	* @remarks No-long-jump zone!!!
2622	*/
2623	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2624	{
2625	int rc = VINF_SUCCESS;
2626	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2627	{
2628	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2629	AssertRCReturn(rc, rc);
2630	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2631	}
2632	return rc;
2633	}
2634
2635
2636	/**
2637	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2638	*
2639	* @returns VBox status code.
2640	* @param pVCpu Pointer to the VMCPU.
2641	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2642	* out-of-sync. Make sure to update the required fields
2643	* before using them.
2644	*
2645	* @remarks No-long-jump zone!!!
2646	*/
2647	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2648	{
2649	int rc = VINF_SUCCESS;
2650	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2651	{
2652	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2653	Let us assert it as such and use 32-bit VMWRITE. */
2654	Assert(!(pMixedCtx->rflags.u64 >> 32));
2655	X86EFLAGS uEFlags = pMixedCtx->eflags;
2656	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2657	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2658
2659	/*
2660	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2661	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2662	*/
2663	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2664	{
2665	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2666	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2667	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2668	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2669	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2670	}
2671
2672	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2673	AssertRCReturn(rc, rc);
2674
2675	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2676	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2677	}
2678	return rc;
2679	}
2680
2681
2682	/**
2683	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2684	*
2685	* @returns VBox status code.
2686	* @param pVCpu Pointer to the VMCPU.
2687	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2688	* out-of-sync. Make sure to update the required fields
2689	* before using them.
2690	*
2691	* @remarks No-long-jump zone!!!
2692	*/
2693	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2694	{
2695	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2696	AssertRCReturn(rc, rc);
2697	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2698	AssertRCReturn(rc, rc);
2699	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2700	AssertRCReturn(rc, rc);
2701	return rc;
2702	}
2703
2704
2705	/**
2706	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2707	* in the VMCS.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM Pointer to the VM.
2711	* @param pVCpu Pointer to the VMCPU.
2712	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2713	* out-of-sync. Make sure to update the required fields
2714	* before using them.
2715	*
2716	* @remarks No-long-jump zone!!!
2717	*/
2718	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2719	{
2720	int rc = VINF_SUCCESS;
2721	PVM pVM = pVCpu->CTX_SUFF(pVM);
2722
2723	/*
2724	* Guest CR0.
2725	* Guest FPU.
2726	*/
2727	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2728	{
2729	Assert(!(pCtx->cr0 >> 32));
2730	uint32_t u32GuestCR0 = pCtx->cr0;
2731
2732	/* The guest's view (read access) of its CR0 is unblemished. */
2733	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2734	AssertRCReturn(rc, rc);
2735	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2736
2737	/* Setup VT-x's view of the guest CR0. */
2738	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2739	if (pVM->hm.s.fNestedPaging)
2740	{
2741	if (CPUMIsGuestPagingEnabledEx(pCtx))
2742	{
2743	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2744	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2745	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2746	}
2747	else
2748	{
2749	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2750	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2751	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2752	}
2753
2754	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2755	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2756	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2757
2758	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2759	AssertRCReturn(rc, rc);
2760	}
2761	else
2762	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2763
2764	/*
2765	* Guest FPU bits.
2766	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2767	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2768	*/
2769	u32GuestCR0 \|= X86_CR0_NE;
2770	bool fInterceptNM = false;
2771	if (CPUMIsGuestFPUStateActive(pVCpu))
2772	{
2773	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2774	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2775	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2776	}
2777	else
2778	{
2779	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2780	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2781	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2782	}
2783
2784	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2785	bool fInterceptMF = false;
2786	if (!(pCtx->cr0 & X86_CR0_NE))
2787	fInterceptMF = true;
2788
2789	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2790	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2791	{
2792	Assert(PDMVmmDevHeapIsEnabled(pVM));
2793	Assert(pVM->hm.s.vmx.pRealModeTSS);
2794	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2795	fInterceptNM = true;
2796	fInterceptMF = true;
2797	}
2798	else
2799	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2800
2801	if (fInterceptNM)
2802	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2803	else
2804	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2805
2806	if (fInterceptMF)
2807	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2808	else
2809	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2810
2811	/* Additional intercepts for debugging, define these yourself explicitly. */
2812	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2813	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2814	\| RT_BIT(X86_XCPT_DB)
2815	\| RT_BIT(X86_XCPT_DE)
2816	\| RT_BIT(X86_XCPT_NM)
2817	\| RT_BIT(X86_XCPT_UD)
2818	\| RT_BIT(X86_XCPT_NP)
2819	\| RT_BIT(X86_XCPT_SS)
2820	\| RT_BIT(X86_XCPT_GP)
2821	\| RT_BIT(X86_XCPT_PF)
2822	\| RT_BIT(X86_XCPT_MF);
2823	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2824	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2825	#endif
2826
2827	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2828
2829	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2830	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2831	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2832	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2833	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2834	else
2835	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2836
2837	u32GuestCR0 \|= uSetCR0;
2838	u32GuestCR0 &= uZapCR0;
2839	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2840
2841	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2842	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2843	AssertRCReturn(rc, rc);
2844	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2845	AssertRCReturn(rc, rc);
2846	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2847
2848	/*
2849	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2850	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2851	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2852	*/
2853	uint32_t u32CR0Mask = 0;
2854	u32CR0Mask = X86_CR0_PE
2855	\| X86_CR0_NE
2856	\| X86_CR0_WP
2857	\| X86_CR0_PG
2858	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2859	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2860	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2861	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2862	u32CR0Mask &= ~X86_CR0_PE;
2863	if (pVM->hm.s.fNestedPaging)
2864	u32CR0Mask &= ~X86_CR0_WP;
2865
2866	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2867	if (fInterceptNM)
2868	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2869	else
2870	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2871
2872	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2873	pVCpu->hm.s.vmx.cr0_mask = u32CR0Mask;
2874	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2875	AssertRCReturn(rc, rc);
2876
2877	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2878	}
2879
2880	/*
2881	* Guest CR2.
2882	* It's always loaded in the assembler code. Nothing to do here.
2883	*/
2884
2885	/*
2886	* Guest CR3.
2887	*/
2888	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2889	{
2890	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2891	if (pVM->hm.s.fNestedPaging)
2892	{
2893	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2894
2895	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2896	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2897	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2898	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2899
2900	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2901	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2902	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2903
2904	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2905	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2906	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2907	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2908
2909	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2910	AssertRCReturn(rc, rc);
2911	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2912
2913	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2914	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2915	{
2916	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2917	if (CPUMIsGuestInPAEModeEx(pCtx))
2918	{
2919	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
2920	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
2921	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
2922	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
2923	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
2924	}
2925
2926	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2927	have Unrestricted Execution to handle the guest when it's not using paging. */
2928	GCPhysGuestCR3 = pCtx->cr3;
2929	}
2930	else
2931	{
2932	/*
2933	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2934	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2935	* EPT takes care of translating it to host-physical addresses.
2936	*/
2937	RTGCPHYS GCPhys;
2938	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2939	Assert(PDMVmmDevHeapIsEnabled(pVM));
2940
2941	/* We obtain it here every time as the guest could have relocated this PCI region. */
2942	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2943	AssertRCReturn(rc, rc);
2944
2945	GCPhysGuestCR3 = GCPhys;
2946	}
2947
2948	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
2949	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2950	}
2951	else
2952	{
2953	/* Non-nested paging case, just use the hypervisor's CR3. */
2954	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2955
2956	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
2957	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
2958	}
2959	AssertRCReturn(rc, rc);
2960
2961	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2962	}
2963
2964	/*
2965	* Guest CR4.
2966	*/
2967	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2968	{
2969	Assert(!(pCtx->cr4 >> 32));
2970	uint32_t u32GuestCR4 = pCtx->cr4;
2971
2972	/* The guest's view of its CR4 is unblemished. */
2973	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2974	AssertRCReturn(rc, rc);
2975	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2976
2977	/* Setup VT-x's view of the guest CR4. */
2978	/*
2979	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2980	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2981	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2982	*/
2983	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2984	{
2985	Assert(pVM->hm.s.vmx.pRealModeTSS);
2986	Assert(PDMVmmDevHeapIsEnabled(pVM));
2987	u32GuestCR4 &= ~X86_CR4_VME;
2988	}
2989
2990	if (pVM->hm.s.fNestedPaging)
2991	{
2992	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2993	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2994	{
2995	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2996	u32GuestCR4 \|= X86_CR4_PSE;
2997	/* Our identity mapping is a 32 bits page directory. */
2998	u32GuestCR4 &= ~X86_CR4_PAE;
2999	}
3000	/* else use guest CR4.*/
3001	}
3002	else
3003	{
3004	/*
3005	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3006	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3007	*/
3008	switch (pVCpu->hm.s.enmShadowMode)
3009	{
3010	case PGMMODE_REAL: /* Real-mode. */
3011	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3012	case PGMMODE_32_BIT: /* 32-bit paging. */
3013	{
3014	u32GuestCR4 &= ~X86_CR4_PAE;
3015	break;
3016	}
3017
3018	case PGMMODE_PAE: /* PAE paging. */
3019	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3020	{
3021	u32GuestCR4 \|= X86_CR4_PAE;
3022	break;
3023	}
3024
3025	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3026	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3027	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3028	break;
3029	#endif
3030	default:
3031	AssertFailed();
3032	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3033	}
3034	}
3035
3036	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3037	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3038	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3039	u32GuestCR4 \|= uSetCR4;
3040	u32GuestCR4 &= uZapCR4;
3041
3042	/* Write VT-x's view of the guest CR4 into the VMCS. */
3043	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3044	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3045	AssertRCReturn(rc, rc);
3046
3047	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3048	uint32_t u32CR4Mask = 0;
3049	u32CR4Mask = X86_CR4_VME
3050	\| X86_CR4_PAE
3051	\| X86_CR4_PGE
3052	\| X86_CR4_PSE
3053	\| X86_CR4_VMXE;
3054	pVCpu->hm.s.vmx.cr4_mask = u32CR4Mask;
3055	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3056	AssertRCReturn(rc, rc);
3057
3058	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3059	}
3060	return rc;
3061	}
3062
3063
3064	/**
3065	* Loads the guest debug registers into the guest-state area in the VMCS.
3066	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3067	*
3068	* @returns VBox status code.
3069	* @param pVCpu Pointer to the VMCPU.
3070	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3071	* out-of-sync. Make sure to update the required fields
3072	* before using them.
3073	*
3074	* @remarks No-long-jump zone!!!
3075	*/
3076	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3077	{
3078	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3079	return VINF_SUCCESS;
3080
3081	#ifdef VBOX_STRICT
3082	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3083	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3084	{
3085	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3086	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3087	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3088	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3089	}
3090	#endif
3091
3092	int rc = VERR_INTERNAL_ERROR_5;
3093	PVM pVM = pVCpu->CTX_SUFF(pVM);
3094	bool fInterceptDB = false;
3095	bool fInterceptMovDRx = false;
3096	if (DBGFIsStepping(pVCpu))
3097	{
3098	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3099	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3100	{
3101	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3102	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3103	AssertRCReturn(rc, rc);
3104	Assert(fInterceptDB == false);
3105	}
3106	else
3107	fInterceptDB = true;
3108	}
3109
3110	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3111	{
3112	if (!CPUMIsHyperDebugStateActive(pVCpu))
3113	{
3114	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3115	AssertRC(rc);
3116	}
3117	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3118	fInterceptMovDRx = true;
3119	}
3120	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3121	{
3122	if (!CPUMIsGuestDebugStateActive(pVCpu))
3123	{
3124	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3125	AssertRC(rc);
3126	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3127	}
3128	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3129	Assert(fInterceptMovDRx == false);
3130	}
3131	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3132	{
3133	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3134	fInterceptMovDRx = true;
3135	}
3136
3137	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3138	if (fInterceptDB)
3139	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3140	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3141	{
3142	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3143	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3144	#endif
3145	}
3146
3147	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3148	if (fInterceptMovDRx)
3149	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3150	else
3151	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3152
3153	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3154	AssertRCReturn(rc, rc);
3155	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3156	AssertRCReturn(rc, rc);
3157
3158	/* The guest's view of its DR7 is unblemished. */
3159	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3160	AssertRCReturn(rc, rc);
3161
3162	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3163	return rc;
3164	}
3165
3166
3167	#ifdef VBOX_STRICT
3168	/**
3169	* Strict function to validate segment registers.
3170	*
3171	* @remarks Requires CR0.
3172	*/
3173	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3174	{
3175	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3176	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3177	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3178
3179	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3180	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3181	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3182	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3183	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3184	{
3185	/* Protected mode checks */
3186	/* CS */
3187	Assert(pCtx->cs.Attr.n.u1Present);
3188	Assert(!(pCtx->cs.Attr.u & 0xf00));
3189	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3190	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3191	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3192	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3193	\|\| (pCtx->cs.Attr.n.u1Granularity));
3194	/* CS cannot be loaded with NULL in protected mode. */
3195	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3196	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3197	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3198	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3199	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3200	else
3201	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3202	/* SS */
3203	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3204	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3205	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3206	if ( !(pCtx->cr0 & X86_CR0_PE)
3207	\|\| pCtx->cs.Attr.n.u4Type == 3)
3208	{
3209	Assert(!pCtx->ss.Attr.n.u2Dpl);
3210	}
3211	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3212	{
3213	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3214	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3215	Assert(pCtx->ss.Attr.n.u1Present);
3216	Assert(!(pCtx->ss.Attr.u & 0xf00));
3217	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3218	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3219	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3220	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3221	\|\| (pCtx->ss.Attr.n.u1Granularity));
3222	}
3223	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3224	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3225	{
3226	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3227	Assert(pCtx->ds.Attr.n.u1Present);
3228	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3229	Assert(!(pCtx->ds.Attr.u & 0xf00));
3230	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3231	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3232	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3233	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3234	\|\| (pCtx->ds.Attr.n.u1Granularity));
3235	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3236	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3237	}
3238	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3239	{
3240	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3241	Assert(pCtx->es.Attr.n.u1Present);
3242	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3243	Assert(!(pCtx->es.Attr.u & 0xf00));
3244	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3245	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3246	\|\| !(pCtx->es.Attr.n.u1Granularity));
3247	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3248	\|\| (pCtx->es.Attr.n.u1Granularity));
3249	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3250	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3251	}
3252	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3253	{
3254	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3255	Assert(pCtx->fs.Attr.n.u1Present);
3256	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3257	Assert(!(pCtx->fs.Attr.u & 0xf00));
3258	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3259	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3260	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3261	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3262	\|\| (pCtx->fs.Attr.n.u1Granularity));
3263	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3264	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3265	}
3266	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3267	{
3268	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3269	Assert(pCtx->gs.Attr.n.u1Present);
3270	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3271	Assert(!(pCtx->gs.Attr.u & 0xf00));
3272	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3273	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3274	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3275	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3276	\|\| (pCtx->gs.Attr.n.u1Granularity));
3277	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3278	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3279	}
3280	/* 64-bit capable CPUs. */
3281	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3282	Assert(!(pCtx->cs.u64Base >> 32));
3283	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3284	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3285	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3286	# endif
3287	}
3288	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3289	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3290	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3291	{
3292	/* Real and v86 mode checks. */
3293	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3294	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3295	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3296	{
3297	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3298	}
3299	else
3300	{
3301	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3302	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3303	}
3304
3305	/* CS */
3306	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3307	Assert(pCtx->cs.u32Limit == 0xffff);
3308	Assert(u32CSAttr == 0xf3);
3309	/* SS */
3310	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3311	Assert(pCtx->ss.u32Limit == 0xffff);
3312	Assert(u32SSAttr == 0xf3);
3313	/* DS */
3314	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3315	Assert(pCtx->ds.u32Limit == 0xffff);
3316	Assert(u32DSAttr == 0xf3);
3317	/* ES */
3318	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3319	Assert(pCtx->es.u32Limit == 0xffff);
3320	Assert(u32ESAttr == 0xf3);
3321	/* FS */
3322	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3323	Assert(pCtx->fs.u32Limit == 0xffff);
3324	Assert(u32FSAttr == 0xf3);
3325	/* GS */
3326	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3327	Assert(pCtx->gs.u32Limit == 0xffff);
3328	Assert(u32GSAttr == 0xf3);
3329	/* 64-bit capable CPUs. */
3330	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3331	Assert(!(pCtx->cs.u64Base >> 32));
3332	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3333	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3334	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3335	# endif
3336	}
3337	}
3338	#endif /* VBOX_STRICT */
3339
3340
3341	/**
3342	* Writes a guest segment register into the guest-state area in the VMCS.
3343	*
3344	* @returns VBox status code.
3345	* @param pVCpu Pointer to the VMCPU.
3346	* @param idxSel Index of the selector in the VMCS.
3347	* @param idxLimit Index of the segment limit in the VMCS.
3348	* @param idxBase Index of the segment base in the VMCS.
3349	* @param idxAccess Index of the access rights of the segment in the VMCS.
3350	* @param pSelReg Pointer to the segment selector.
3351	* @param pCtx Pointer to the guest-CPU context.
3352	*
3353	* @remarks No-long-jump zone!!!
3354	*/
3355	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3356	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3357	{
3358	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3359	AssertRCReturn(rc, rc);
3360	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3361	AssertRCReturn(rc, rc);
3362	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3363	AssertRCReturn(rc, rc);
3364
3365	uint32_t u32Access = pSelReg->Attr.u;
3366	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3367	{
3368	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3369	u32Access = 0xf3;
3370	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3371	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3372	}
3373	else
3374	{
3375	/*
3376	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3377	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3378	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3379	* loaded in protected-mode have their attribute as 0.
3380	*/
3381	if (!u32Access)
3382	u32Access = HMVMX_SEL_UNUSABLE;
3383	}
3384
3385	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3386	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3387	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3388
3389	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3390	AssertRCReturn(rc, rc);
3391	return rc;
3392	}
3393
3394
3395	/**
3396	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3397	* into the guest-state area in the VMCS.
3398	*
3399	* @returns VBox status code.
3400	* @param pVM Pointer to the VM.
3401	* @param pVCPU Pointer to the VMCPU.
3402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403	* out-of-sync. Make sure to update the required fields
3404	* before using them.
3405	*
3406	* @remarks Requires CR0 (strict builds validation).
3407	* @remarks No-long-jump zone!!!
3408	*/
3409	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3410	{
3411	int rc = VERR_INTERNAL_ERROR_5;
3412	PVM pVM = pVCpu->CTX_SUFF(pVM);
3413
3414	/*
3415	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3416	*/
3417	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3418	{
3419	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3420	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3421	{
3422	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3423	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3424	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3425	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3426	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3427	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3428	}
3429
3430	#ifdef VBOX_WITH_REM
3431	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3432	{
3433	Assert(pVM->hm.s.vmx.pRealModeTSS);
3434	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3435	if ( pVCpu->hm.s.vmx.fWasInRealMode
3436	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3437	{
3438	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3439	in real-mode (e.g. OpenBSD 4.0) */
3440	REMFlushTBs(pVM);
3441	Log(("Load: Switch to protected mode detected!\n"));
3442	pVCpu->hm.s.vmx.fWasInRealMode = false;
3443	}
3444	}
3445	#endif
3446	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3447	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3448	AssertRCReturn(rc, rc);
3449	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3450	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3451	AssertRCReturn(rc, rc);
3452	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3453	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3454	AssertRCReturn(rc, rc);
3455	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3456	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3457	AssertRCReturn(rc, rc);
3458	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3459	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3460	AssertRCReturn(rc, rc);
3461	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3462	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3463	AssertRCReturn(rc, rc);
3464
3465	#ifdef VBOX_STRICT
3466	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3467	#endif
3468	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3469	}
3470
3471	/*
3472	* Guest TR.
3473	*/
3474	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3475	{
3476	/*
3477	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3478	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3479	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3480	*/
3481	uint16_t u16Sel = 0;
3482	uint32_t u32Limit = 0;
3483	uint64_t u64Base = 0;
3484	uint32_t u32AccessRights = 0;
3485
3486	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3487	{
3488	u16Sel = pMixedCtx->tr.Sel;
3489	u32Limit = pMixedCtx->tr.u32Limit;
3490	u64Base = pMixedCtx->tr.u64Base;
3491	u32AccessRights = pMixedCtx->tr.Attr.u;
3492	}
3493	else
3494	{
3495	Assert(pVM->hm.s.vmx.pRealModeTSS);
3496	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3497
3498	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3499	RTGCPHYS GCPhys;
3500	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3501	AssertRCReturn(rc, rc);
3502
3503	X86DESCATTR DescAttr;
3504	DescAttr.u = 0;
3505	DescAttr.n.u1Present = 1;
3506	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3507
3508	u16Sel = 0;
3509	u32Limit = HM_VTX_TSS_SIZE;
3510	u64Base = GCPhys; /* in real-mode phys = virt. */
3511	u32AccessRights = DescAttr.u;
3512	}
3513
3514	/* Validate. */
3515	Assert(!(u16Sel & RT_BIT(2)));
3516	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3517	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3518	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3519	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3520	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3521	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3522	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3523	Assert( (u32Limit & 0xfff) == 0xfff
3524	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3525	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3526	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3527
3528	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3529	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3530	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3531	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3532
3533	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3534	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3535	}
3536
3537	/*
3538	* Guest GDTR.
3539	*/
3540	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3541	{
3542	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3543	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3544
3545	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3546	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3547	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3548	}
3549
3550	/*
3551	* Guest LDTR.
3552	*/
3553	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3554	{
3555	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3556	uint32_t u32Access = 0;
3557	if (!pMixedCtx->ldtr.Attr.u)
3558	u32Access = HMVMX_SEL_UNUSABLE;
3559	else
3560	u32Access = pMixedCtx->ldtr.Attr.u;
3561
3562	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3563	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3564	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3565	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3566
3567	/* Validate. */
3568	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3569	{
3570	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3571	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3572	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3573	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3574	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3575	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3576	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3577	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3578	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3579	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3580	}
3581
3582	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3583	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3584	}
3585
3586	/*
3587	* Guest IDTR.
3588	*/
3589	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3590	{
3591	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3592	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3593
3594	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3595	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3596	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3597	}
3598
3599	return VINF_SUCCESS;
3600	}
3601
3602
3603	/**
3604	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3605	* areas. These MSRs will automatically be loaded to the host CPU on every
3606	* successful VM entry and stored from the host CPU on every successful VM exit.
3607	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3608	*
3609	* @returns VBox status code.
3610	* @param pVCpu Pointer to the VMCPU.
3611	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3612	* out-of-sync. Make sure to update the required fields
3613	* before using them.
3614	*
3615	* @remarks No-long-jump zone!!!
3616	*/
3617	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3618	{
3619	AssertPtr(pVCpu);
3620	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3621
3622	/*
3623	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3624	*/
3625	int rc = VINF_SUCCESS;
3626	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3627	{
3628	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3629	PVM pVM = pVCpu->CTX_SUFF(pVM);
3630	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3631	uint32_t cGuestMsrs = 0;
3632
3633	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3634	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3635	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3636	if (fSupportsNX \|\| fSupportsLongMode)
3637	{
3638	/** @todo support save IA32_EFER, i.e.
3639	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, in which case the
3640	* guest EFER need not be part of the VM-entry MSR-load area. */
3641	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3642	pGuestMsr->u32Reserved = 0;
3643	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3644	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3645	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3646	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3647	pGuestMsr++; cGuestMsrs++;
3648	if (fSupportsLongMode)
3649	{
3650	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3651	pGuestMsr->u32Reserved = 0;
3652	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3653	pGuestMsr++; cGuestMsrs++;
3654	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3655	pGuestMsr->u32Reserved = 0;
3656	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3657	pGuestMsr++; cGuestMsrs++;
3658	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3659	pGuestMsr->u32Reserved = 0;
3660	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3661	pGuestMsr++; cGuestMsrs++;
3662	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3663	pGuestMsr->u32Reserved = 0;
3664	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3665	pGuestMsr++; cGuestMsrs++;
3666	}
3667	}
3668
3669	/*
3670	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3671	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3672	*/
3673	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3674	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3675	{
3676	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3677	pGuestMsr->u32Reserved = 0;
3678	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3679	AssertRCReturn(rc, rc);
3680	pGuestMsr++; cGuestMsrs++;
3681	}
3682
3683	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3684	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3685	{
3686	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3687	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3688	}
3689
3690	/* Update the VCPU's copy of the guest MSR count. */
3691	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3692	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3693	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3694	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3695
3696	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3697	}
3698
3699	/*
3700	* Guest Sysenter MSRs.
3701	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3702	* VM-exits on WRMSRs for these MSRs.
3703	*/
3704	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3705	{
3706	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3707	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3708	}
3709	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3710	{
3711	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3712	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3713	}
3714	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3715	{
3716	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3717	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3718	}
3719
3720	return rc;
3721	}
3722
3723
3724	/**
3725	* Loads the guest activity state into the guest-state area in the VMCS.
3726	*
3727	* @returns VBox status code.
3728	* @param pVCpu Pointer to the VMCPU.
3729	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3730	* out-of-sync. Make sure to update the required fields
3731	* before using them.
3732	*
3733	* @remarks No-long-jump zone!!!
3734	*/
3735	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3736	{
3737	/** @todo See if we can make use of other states, e.g.
3738	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3739	int rc = VINF_SUCCESS;
3740	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3741	{
3742	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3743	AssertRCReturn(rc, rc);
3744	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3745	}
3746	return rc;
3747	}
3748
3749
3750	/**
3751	* Sets up the appropriate function to run guest code.
3752	*
3753	* @returns VBox status code.
3754	* @param pVCpu Pointer to the VMCPU.
3755	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3756	* out-of-sync. Make sure to update the required fields
3757	* before using them.
3758	*
3759	* @remarks No-long-jump zone!!!
3760	*/
3761	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3762	{
3763	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3764	{
3765	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3766	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3767	#endif
3768	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3769	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3770	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3771	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3772	#else
3773	/* 64-bit host or hybrid host. */
3774	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3775	#endif
3776	}
3777	else
3778	{
3779	/* Guest is not in long mode, use the 32-bit handler. */
3780	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3781	}
3782	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3783	return VINF_SUCCESS;
3784	}
3785
3786
3787	/**
3788	* Wrapper for running the guest code in VT-x.
3789	*
3790	* @returns VBox strict status code.
3791	* @param pVM Pointer to the VM.
3792	* @param pVCpu Pointer to the VMCPU.
3793	* @param pCtx Pointer to the guest-CPU context.
3794	*
3795	* @remarks No-long-jump zone!!!
3796	*/
3797	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3798	{
3799	/*
3800	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3801	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3802	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3803	*/
3804	#ifdef VBOX_WITH_KERNEL_USING_XMM
3805	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3806	#else
3807	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3808	#endif
3809	}
3810
3811
3812	/**
3813	* Report world-switch error and dump some useful debug info.
3814	*
3815	* @param pVM Pointer to the VM.
3816	* @param pVCpu Pointer to the VMCPU.
3817	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3818	* @param pCtx Pointer to the guest-CPU context.
3819	* @param pVmxTransient Pointer to the VMX transient structure (only
3820	* exitReason updated).
3821	*/
3822	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3823	{
3824	Assert(pVM);
3825	Assert(pVCpu);
3826	Assert(pCtx);
3827	Assert(pVmxTransient);
3828	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3829
3830	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3831	switch (rcVMRun)
3832	{
3833	case VERR_VMX_INVALID_VMXON_PTR:
3834	AssertFailed();
3835	break;
3836	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3837	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3838	{
3839	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3840	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3841	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3842	AssertRC(rc);
3843
3844	#ifdef VBOX_STRICT
3845	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3846	pVmxTransient->uExitReason));
3847	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3848	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3849	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3850	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3851	else
3852	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3853
3854	/* VMX control bits. */
3855	uint32_t u32Val;
3856	uint64_t u64Val;
3857	HMVMXHCUINTREG uHCReg;
3858	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3859	Log(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3860	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3861	Log(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3862	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3863	Log(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3864	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3865	Log(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3866	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3867	Log(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3868	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3869	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3870	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3871	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3872	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3873	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3874	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3875	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3876	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3877	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3878	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3879	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3880	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3881	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3882	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3883	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3884	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3885	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3886	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3887	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3888	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3889	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3890	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3891	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3892	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3893	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3894	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3895	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3896	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3897	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3898	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3899	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3900
3901	/* Guest bits. */
3902	RTGCUINTREG uGCReg;
3903	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3904	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3905	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3906	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3907	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3908	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3909	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3910	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3911
3912	/* Host bits. */
3913	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3914	Log(("Host CR0 %#RHr\n", uHCReg));
3915	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3916	Log(("Host CR3 %#RHr\n", uHCReg));
3917	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3918	Log(("Host CR4 %#RHr\n", uHCReg));
3919
3920	RTGDTR HostGdtr;
3921	PCX86DESCHC pDesc;
3922	ASMGetGDTR(&HostGdtr);
3923	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3924	Log(("Host CS %#08x\n", u32Val));
3925	if (u32Val < HostGdtr.cbGdt)
3926	{
3927	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3928	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3929	}
3930
3931	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3932	Log(("Host DS %#08x\n", u32Val));
3933	if (u32Val < HostGdtr.cbGdt)
3934	{
3935	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3936	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3937	}
3938
3939	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3940	Log(("Host ES %#08x\n", u32Val));
3941	if (u32Val < HostGdtr.cbGdt)
3942	{
3943	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3944	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3945	}
3946
3947	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3948	Log(("Host FS %#08x\n", u32Val));
3949	if (u32Val < HostGdtr.cbGdt)
3950	{
3951	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3952	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3953	}
3954
3955	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3956	Log(("Host GS %#08x\n", u32Val));
3957	if (u32Val < HostGdtr.cbGdt)
3958	{
3959	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3960	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3961	}
3962
3963	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3964	Log(("Host SS %#08x\n", u32Val));
3965	if (u32Val < HostGdtr.cbGdt)
3966	{
3967	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3968	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3969	}
3970
3971	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3972	Log(("Host TR %#08x\n", u32Val));
3973	if (u32Val < HostGdtr.cbGdt)
3974	{
3975	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3976	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3977	}
3978
3979	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3980	Log(("Host TR Base %#RHv\n", uHCReg));
3981	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3982	Log(("Host GDTR Base %#RHv\n", uHCReg));
3983	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3984	Log(("Host IDTR Base %#RHv\n", uHCReg));
3985	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3986	Log(("Host SYSENTER CS %#08x\n", u32Val));
3987	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3988	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3989	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3990	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3991	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3992	Log(("Host RSP %#RHv\n", uHCReg));
3993	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3994	Log(("Host RIP %#RHv\n", uHCReg));
3995	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3996	if (HMVMX_IS_64BIT_HOST_MODE())
3997	{
3998	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3999	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4000	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4001	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4002	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4003	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4004	}
4005	# endif
4006	#endif /* VBOX_STRICT */
4007	break;
4008	}
4009
4010	default:
4011	/* Impossible */
4012	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4013	break;
4014	}
4015	NOREF(pVM);
4016	}
4017
4018
4019	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4020	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4021	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4022	#endif
4023	#ifdef VBOX_STRICT
4024	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4025	{
4026	switch (idxField)
4027	{
4028	case VMX_VMCS_GUEST_RIP:
4029	case VMX_VMCS_GUEST_RSP:
4030	case VMX_VMCS_GUEST_DR7:
4031	case VMX_VMCS_GUEST_SYSENTER_EIP:
4032	case VMX_VMCS_GUEST_SYSENTER_ESP:
4033	case VMX_VMCS_GUEST_GDTR_BASE:
4034	case VMX_VMCS_GUEST_IDTR_BASE:
4035	case VMX_VMCS_GUEST_CS_BASE:
4036	case VMX_VMCS_GUEST_DS_BASE:
4037	case VMX_VMCS_GUEST_ES_BASE:
4038	case VMX_VMCS_GUEST_FS_BASE:
4039	case VMX_VMCS_GUEST_GS_BASE:
4040	case VMX_VMCS_GUEST_SS_BASE:
4041	case VMX_VMCS_GUEST_LDTR_BASE:
4042	case VMX_VMCS_GUEST_TR_BASE:
4043	case VMX_VMCS_GUEST_CR3:
4044	return true;
4045	}
4046	return false;
4047	}
4048
4049	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4050	{
4051	switch (idxField)
4052	{
4053	/* Read-only fields. */
4054	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4055	return true;
4056	}
4057	/* Remaining readable fields should also be writable. */
4058	return hmR0VmxIsValidWriteField(idxField);
4059	}
4060	#endif /* VBOX_STRICT */
4061
4062	/**
4063	* Executes the specified handler in 64-bit mode.
4064	*
4065	* @returns VBox status code.
4066	* @param pVM Pointer to the VM.
4067	* @param pVCpu Pointer to the VMCPU.
4068	* @param pCtx Pointer to the guest CPU context.
4069	* @param enmOp The operation to perform.
4070	* @param cbParam Number of parameters.
4071	* @param paParam Array of 32-bit parameters.
4072	*/
4073	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4074	uint32_t *paParam)
4075	{
4076	int rc, rc2;
4077	PHMGLOBLCPUINFO pCpu;
4078	RTHCPHYS HCPhysCpuPage;
4079	RTCCUINTREG uOldEFlags;
4080
4081	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4082	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4083	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4084	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4085
4086	#ifdef VBOX_STRICT
4087	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4088	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4089
4090	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4091	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4092	#endif
4093
4094	/* Disable interrupts. */
4095	uOldEFlags = ASMIntDisableFlags();
4096
4097	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4098	RTCPUID idHostCpu = RTMpCpuId();
4099	CPUMR0SetLApic(pVM, idHostCpu);
4100	#endif
4101
4102	pCpu = HMR0GetCurrentCpu();
4103	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4104
4105	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4106	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4107
4108	/* Leave VMX Root Mode. */
4109	VMXDisable();
4110
4111	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4112
4113	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4114	CPUMSetHyperEIP(pVCpu, enmOp);
4115	for (int i = (int)cbParam - 1; i >= 0; i--)
4116	CPUMPushHyper(pVCpu, paParam[i]);
4117
4118	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4119
4120	/* Call the switcher. */
4121	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4122	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4123
4124	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4125	/* Make sure the VMX instructions don't cause #UD faults. */
4126	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4127
4128	/* Re-enter VMX Root Mode */
4129	rc2 = VMXEnable(HCPhysCpuPage);
4130	if (RT_FAILURE(rc2))
4131	{
4132	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4133	ASMSetFlags(uOldEFlags);
4134	return rc2;
4135	}
4136
4137	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4138	AssertRC(rc2);
4139	Assert(!(ASMGetFlags() & X86_EFL_IF));
4140	ASMSetFlags(uOldEFlags);
4141	return rc;
4142	}
4143
4144
4145	/**
4146	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4147	* supporting 64-bit guests.
4148	*
4149	* @returns VBox status code.
4150	* @param fResume Whether to VMLAUNCH or VMRESUME.
4151	* @param pCtx Pointer to the guest-CPU context.
4152	* @param pCache Pointer to the VMCS cache.
4153	* @param pVM Pointer to the VM.
4154	* @param pVCpu Pointer to the VMCPU.
4155	*/
4156	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4157	{
4158	uint32_t aParam[6];
4159	PHMGLOBLCPUINFO pCpu = NULL;
4160	RTHCPHYS HCPhysCpuPage = 0;
4161	int rc = VERR_INTERNAL_ERROR_5;
4162
4163	pCpu = HMR0GetCurrentCpu();
4164	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4165
4166	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4167	pCache->uPos = 1;
4168	pCache->interPD = PGMGetInterPaeCR3(pVM);
4169	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4170	#endif
4171
4172	#ifdef VBOX_STRICT
4173	pCache->TestIn.HCPhysCpuPage = 0;
4174	pCache->TestIn.HCPhysVmcs = 0;
4175	pCache->TestIn.pCache = 0;
4176	pCache->TestOut.HCPhysVmcs = 0;
4177	pCache->TestOut.pCache = 0;
4178	pCache->TestOut.pCtx = 0;
4179	pCache->TestOut.eflags = 0;
4180	#endif
4181
4182	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4183	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4184	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4185	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4186	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4187	aParam[5] = 0;
4188
4189	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4190	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4191	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4192	#endif
4193	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4194
4195	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4196	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4197	Assert(pCtx->dr[4] == 10);
4198	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4199	#endif
4200
4201	#ifdef VBOX_STRICT
4202	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4203	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4204	pVCpu->hm.s.vmx.HCPhysVmcs));
4205	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4206	pCache->TestOut.HCPhysVmcs));
4207	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4208	pCache->TestOut.pCache));
4209	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4210	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4211	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4212	pCache->TestOut.pCtx));
4213	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4214	#endif
4215	return rc;
4216	}
4217
4218
4219	/**
4220	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4221	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4222	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4223	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4224	*
4225	* @returns VBox status code.
4226	* @param pVM Pointer to the VM.
4227	* @param pVCpu Pointer to the VMCPU.
4228	*/
4229	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4230	{
4231	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4232	{ \
4233	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4234	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4235	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4236	++cReadFields; \
4237	}
4238
4239	AssertPtr(pVM);
4240	AssertPtr(pVCpu);
4241	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4242	uint32_t cReadFields = 0;
4243
4244	/* Guest-natural selector base fields */
4245	#if 0
4246	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4249	#endif
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4252	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4257	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4263	#if 0
4264	/* Unused natural width guest-state fields. */
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4267	#endif
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4270
4271	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4272	#if 0
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4282	#endif
4283
4284	/* Natural width guest-state fields. */
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4286	#if 0
4287	/* Currently unused field. */
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4289	#endif
4290
4291	if (pVM->hm.s.fNestedPaging)
4292	{
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4294	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4295	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4296	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4297	}
4298	else
4299	{
4300	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4301	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4302	}
4303
4304	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4305	return VINF_SUCCESS;
4306	}
4307
4308
4309	/**
4310	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4311	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4312	* darwin, running 64-bit guests).
4313	*
4314	* @returns VBox status code.
4315	* @param pVCpu Pointer to the VMCPU.
4316	* @param idxField The VMCS field encoding.
4317	* @param u64Val 16, 32 or 64 bits value.
4318	*/
4319	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4320	{
4321	int rc;
4322	switch (idxField)
4323	{
4324	/*
4325	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4326	*/
4327	/* 64-bit Control fields. */
4328	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4329	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4330	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4331	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4332	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4333	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4334	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4335	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4336	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4337	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4338	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4339	case VMX_VMCS64_CTRL_EPTP_FULL:
4340	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4341	/* 64-bit Guest-state fields. */
4342	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4343	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4344	case VMX_VMCS64_GUEST_PAT_FULL:
4345	case VMX_VMCS64_GUEST_EFER_FULL:
4346	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4347	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4348	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4349	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4350	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4351	/* 64-bit Host-state fields. */
4352	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4353	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4354	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4355	{
4356	rc = VMXWriteVmcs32(idxField, u64Val);
4357	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4358	break;
4359	}
4360
4361	/*
4362	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4363	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4364	*/
4365	/* Natural-width Guest-state fields. */
4366	case VMX_VMCS_GUEST_CR3:
4367	case VMX_VMCS_GUEST_ES_BASE:
4368	case VMX_VMCS_GUEST_CS_BASE:
4369	case VMX_VMCS_GUEST_SS_BASE:
4370	case VMX_VMCS_GUEST_DS_BASE:
4371	case VMX_VMCS_GUEST_FS_BASE:
4372	case VMX_VMCS_GUEST_GS_BASE:
4373	case VMX_VMCS_GUEST_LDTR_BASE:
4374	case VMX_VMCS_GUEST_TR_BASE:
4375	case VMX_VMCS_GUEST_GDTR_BASE:
4376	case VMX_VMCS_GUEST_IDTR_BASE:
4377	case VMX_VMCS_GUEST_DR7:
4378	case VMX_VMCS_GUEST_RSP:
4379	case VMX_VMCS_GUEST_RIP:
4380	case VMX_VMCS_GUEST_SYSENTER_ESP:
4381	case VMX_VMCS_GUEST_SYSENTER_EIP:
4382	{
4383	if (!(u64Val >> 32))
4384	{
4385	/* If this field is 64-bit, VT-x will zero out the top bits. */
4386	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4387	}
4388	else
4389	{
4390	/* Assert that only the 32->64 switcher case should ever come here. */
4391	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4392	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4393	}
4394	break;
4395	}
4396
4397	default:
4398	{
4399	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4400	rc = VERR_INVALID_PARAMETER;
4401	break;
4402	}
4403	}
4404	AssertRCReturn(rc, rc);
4405	return rc;
4406	}
4407
4408
4409	/**
4410	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4411	* hosts (except darwin) for 64-bit guests.
4412	*
4413	* @param pVCpu Pointer to the VMCPU.
4414	* @param idxField The VMCS field encoding.
4415	* @param u64Val 16, 32 or 64 bits value.
4416	*/
4417	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4418	{
4419	AssertPtr(pVCpu);
4420	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4421
4422	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4423	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4424
4425	/* Make sure there are no duplicates. */
4426	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4427	{
4428	if (pCache->Write.aField[i] == idxField)
4429	{
4430	pCache->Write.aFieldVal[i] = u64Val;
4431	return VINF_SUCCESS;
4432	}
4433	}
4434
4435	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4436	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4437	pCache->Write.cValidEntries++;
4438	return VINF_SUCCESS;
4439	}
4440
4441	/* Enable later when the assembly code uses these as callbacks. */
4442	#if 0
4443	/*
4444	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4445	*
4446	* @param pVCpu Pointer to the VMCPU.
4447	* @param pCache Pointer to the VMCS cache.
4448	*
4449	* @remarks No-long-jump zone!!!
4450	*/
4451	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4452	{
4453	AssertPtr(pCache);
4454	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4455	{
4456	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4457	AssertRC(rc);
4458	}
4459	pCache->Write.cValidEntries = 0;
4460	}
4461
4462
4463	/**
4464	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4465	*
4466	* @param pVCpu Pointer to the VMCPU.
4467	* @param pCache Pointer to the VMCS cache.
4468	*
4469	* @remarks No-long-jump zone!!!
4470	*/
4471	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4472	{
4473	AssertPtr(pCache);
4474	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4475	{
4476	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4477	AssertRC(rc);
4478	}
4479	}
4480	#endif
4481	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4482
4483
4484	/**
4485	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4486	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4487	* timer.
4488	*
4489	* @returns VBox status code.
4490	* @param pVCpu Pointer to the VMCPU.
4491	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4492	* out-of-sync. Make sure to update the required fields
4493	* before using them.
4494	* @remarks No-long-jump zone!!!
4495	*/
4496	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4497	{
4498	int rc = VERR_INTERNAL_ERROR_5;
4499	bool fOffsettedTsc = false;
4500	PVM pVM = pVCpu->CTX_SUFF(pVM);
4501	if (pVM->hm.s.vmx.fUsePreemptTimer)
4502	{
4503	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4504
4505	/* Make sure the returned values have sane upper and lower boundaries. */
4506	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4507	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4508	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4509	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4510
4511	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4512	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4513	}
4514	else
4515	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4516
4517	if (fOffsettedTsc)
4518	{
4519	uint64_t u64CurTSC = ASMReadTSC();
4520	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4521	{
4522	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4523	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4524
4525	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4526	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4527	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4528	}
4529	else
4530	{
4531	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4532	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4533	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4534	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4535	}
4536	}
4537	else
4538	{
4539	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4540	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4541	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4542	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4543	}
4544	}
4545
4546
4547	/**
4548	* Determines if an exception is a contributory exception. Contributory
4549	* exceptions are ones which can cause double-faults. Page-fault is
4550	* intentionally not included here as it's a conditional contributory exception.
4551	*
4552	* @returns true if the exception is contributory, false otherwise.
4553	* @param uVector The exception vector.
4554	*/
4555	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4556	{
4557	switch (uVector)
4558	{
4559	case X86_XCPT_GP:
4560	case X86_XCPT_SS:
4561	case X86_XCPT_NP:
4562	case X86_XCPT_TS:
4563	case X86_XCPT_DE:
4564	return true;
4565	default:
4566	break;
4567	}
4568	return false;
4569	}
4570
4571
4572	/**
4573	* Sets an event as a pending event to be injected into the guest.
4574	*
4575	* @param pVCpu Pointer to the VMCPU.
4576	* @param u32IntrInfo The VM-entry interruption-information field.
4577	* @param cbInstr The VM-entry instruction length in bytes (for software
4578	* interrupts, exceptions and privileged software
4579	* exceptions).
4580	* @param u32ErrCode The VM-entry exception error code.
4581	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4582	* page-fault.
4583	*/
4584	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4585	RTGCUINTPTR GCPtrFaultAddress)
4586	{
4587	Assert(!pVCpu->hm.s.Event.fPending);
4588	pVCpu->hm.s.Event.fPending = true;
4589	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4590	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4591	pVCpu->hm.s.Event.cbInstr = cbInstr;
4592	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4593	}
4594
4595
4596	/**
4597	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4598	*
4599	* @param pVCpu Pointer to the VMCPU.
4600	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4601	* out-of-sync. Make sure to update the required fields
4602	* before using them.
4603	*/
4604	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4605	{
4606	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4607	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4608	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4609	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4610	}
4611
4612
4613	/**
4614	* Handle a condition that occurred while delivering an event through the guest
4615	* IDT.
4616	*
4617	* @returns VBox status code (informational error codes included).
4618	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4619	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4620	* continue execution of the guest which will delivery the #DF.
4621	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4622	*
4623	* @param pVCpu Pointer to the VMCPU.
4624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4625	* out-of-sync. Make sure to update the required fields
4626	* before using them.
4627	* @param pVmxTransient Pointer to the VMX transient structure.
4628	*
4629	* @remarks No-long-jump zone!!!
4630	*/
4631	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4632	{
4633	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4634	AssertRC(rc);
4635	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4636	{
4637	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4638	AssertRCReturn(rc, rc);
4639
4640	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4641	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4642	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4643
4644	typedef enum
4645	{
4646	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4647	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4648	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4649	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4650	} VMXREFLECTXCPT;
4651
4652	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4653	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4654	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4655	{
4656	enmReflect = VMXREFLECTXCPT_XCPT;
4657	#ifdef VBOX_STRICT
4658	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4659	&& uExitVector == X86_XCPT_PF)
4660	{
4661	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4662	}
4663	#endif
4664	if ( uExitVector == X86_XCPT_PF
4665	&& uIdtVector == X86_XCPT_PF)
4666	{
4667	pVmxTransient->fVectoringPF = true;
4668	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4669	}
4670	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4671	&& hmR0VmxIsContributoryXcpt(uExitVector)
4672	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4673	\|\| uIdtVector == X86_XCPT_PF))
4674	{
4675	enmReflect = VMXREFLECTXCPT_DF;
4676	}
4677	else if (uIdtVector == X86_XCPT_DF)
4678	enmReflect = VMXREFLECTXCPT_TF;
4679	}
4680	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4681	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4682	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4683	{
4684	/*
4685	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4686	* (whatever they are) as they reoccur when restarting the instruction.
4687	*/
4688	enmReflect = VMXREFLECTXCPT_XCPT;
4689	}
4690
4691	switch (enmReflect)
4692	{
4693	case VMXREFLECTXCPT_XCPT:
4694	{
4695	uint32_t u32ErrCode = 0;
4696	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4697	{
4698	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4699	AssertRCReturn(rc, rc);
4700	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4701	}
4702
4703	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4704	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4705	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4706	rc = VINF_SUCCESS;
4707	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4708	pVCpu->hm.s.Event.u32ErrCode));
4709	break;
4710	}
4711
4712	case VMXREFLECTXCPT_DF:
4713	{
4714	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4715	rc = VINF_VMX_DOUBLE_FAULT;
4716	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4717	uIdtVector, uExitVector));
4718	break;
4719	}
4720
4721	case VMXREFLECTXCPT_TF:
4722	{
4723	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4724	rc = VINF_EM_RESET;
4725	break;
4726	}
4727
4728	default:
4729	Assert(rc == VINF_SUCCESS);
4730	break;
4731	}
4732	}
4733	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4734	return rc;
4735	}
4736
4737
4738	/**
4739	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4740	*
4741	* @returns VBox status code.
4742	* @param pVCpu Pointer to the VMCPU.
4743	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4744	* out-of-sync. Make sure to update the required fields
4745	* before using them.
4746	*
4747	* @remarks No-long-jump zone!!!
4748	*/
4749	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4750	{
4751	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4752	{
4753	uint32_t uVal = 0;
4754	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4755	AssertRCReturn(rc, rc);
4756	uint32_t uShadow = 0;
4757	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4758	AssertRCReturn(rc, rc);
4759
4760	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4761	CPUMSetGuestCR0(pVCpu, uVal);
4762	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4763	}
4764	return VINF_SUCCESS;
4765	}
4766
4767
4768	/**
4769	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4770	*
4771	* @returns VBox status code.
4772	* @param pVCpu Pointer to the VMCPU.
4773	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4774	* out-of-sync. Make sure to update the required fields
4775	* before using them.
4776	*
4777	* @remarks No-long-jump zone!!!
4778	*/
4779	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4780	{
4781	int rc = VINF_SUCCESS;
4782	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4783	{
4784	uint32_t uVal = 0;
4785	uint32_t uShadow = 0;
4786	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4787	AssertRCReturn(rc, rc);
4788	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4789	AssertRCReturn(rc, rc);
4790
4791	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4792	CPUMSetGuestCR4(pVCpu, uVal);
4793	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4794	}
4795	return rc;
4796	}
4797
4798
4799	/**
4800	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4801	*
4802	* @returns VBox status code.
4803	* @param pVCpu Pointer to the VMCPU.
4804	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4805	* out-of-sync. Make sure to update the required fields
4806	* before using them.
4807	*
4808	* @remarks No-long-jump zone!!!
4809	*/
4810	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811	{
4812	int rc = VINF_SUCCESS;
4813	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4814	{
4815	RTGCUINTREG uVal = 0;
4816	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4817	AssertRCReturn(rc, rc);
4818
4819	pMixedCtx->rip = uVal;
4820	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4821	}
4822	return rc;
4823	}
4824
4825
4826	/**
4827	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4828	*
4829	* @returns VBox status code.
4830	* @param pVCpu Pointer to the VMCPU.
4831	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4832	* out-of-sync. Make sure to update the required fields
4833	* before using them.
4834	*
4835	* @remarks No-long-jump zone!!!
4836	*/
4837	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838	{
4839	int rc = VINF_SUCCESS;
4840	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4841	{
4842	RTGCUINTREG uVal = 0;
4843	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4844	AssertRCReturn(rc, rc);
4845
4846	pMixedCtx->rsp = uVal;
4847	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4848	}
4849	return rc;
4850	}
4851
4852
4853	/**
4854	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4855	*
4856	* @returns VBox status code.
4857	* @param pVCpu Pointer to the VMCPU.
4858	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4859	* out-of-sync. Make sure to update the required fields
4860	* before using them.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4865	{
4866	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4867	{
4868	uint32_t uVal = 0;
4869	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4870	AssertRCReturn(rc, rc);
4871
4872	pMixedCtx->eflags.u32 = uVal;
4873	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4874	{
4875	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4876	Log(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4877
4878	pMixedCtx->eflags.Bits.u1VM = 0;
4879	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4880	}
4881
4882	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4883	}
4884	return VINF_SUCCESS;
4885	}
4886
4887
4888	/**
4889	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4890	* guest-CPU context.
4891	*/
4892	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4893	{
4894	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4895	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4896	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4897	return rc;
4898	}
4899
4900
4901	/**
4902	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4903	* from the guest-state area in the VMCS.
4904	*
4905	* @param pVCpu Pointer to the VMCPU.
4906	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4907	* out-of-sync. Make sure to update the required fields
4908	* before using them.
4909	*
4910	* @remarks No-long-jump zone!!!
4911	*/
4912	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4913	{
4914	uint32_t uIntrState = 0;
4915	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4916	AssertRC(rc);
4917
4918	if (!uIntrState)
4919	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4920	else
4921	{
4922	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4923	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4924	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4925	AssertRC(rc);
4926	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4927	AssertRC(rc);
4928
4929	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4930	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4931	}
4932	}
4933
4934
4935	/**
4936	* Saves the guest's activity state.
4937	*
4938	* @returns VBox status code.
4939	* @param pVCpu Pointer to the VMCPU.
4940	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4941	* out-of-sync. Make sure to update the required fields
4942	* before using them.
4943	*
4944	* @remarks No-long-jump zone!!!
4945	*/
4946	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4947	{
4948	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4949	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4950	return VINF_SUCCESS;
4951	}
4952
4953
4954	/**
4955	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4956	* the current VMCS into the guest-CPU context.
4957	*
4958	* @returns VBox status code.
4959	* @param pVCpu Pointer to the VMCPU.
4960	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4961	* out-of-sync. Make sure to update the required fields
4962	* before using them.
4963	*
4964	* @remarks No-long-jump zone!!!
4965	*/
4966	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4967	{
4968	int rc = VINF_SUCCESS;
4969	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4970	{
4971	uint32_t u32Val = 0;
4972	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4973	pMixedCtx->SysEnter.cs = u32Val;
4974	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4975	}
4976
4977	RTGCUINTREG uGCVal = 0;
4978	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4979	{
4980	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4981	pMixedCtx->SysEnter.eip = uGCVal;
4982	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4983	}
4984	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4985	{
4986	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4987	pMixedCtx->SysEnter.esp = uGCVal;
4988	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4989	}
4990	return rc;
4991	}
4992
4993
4994	/**
4995	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4996	* context.
4997	*
4998	* @returns VBox status code.
4999	* @param pVCpu Pointer to the VMCPU.
5000	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5001	* out-of-sync. Make sure to update the required fields
5002	* before using them.
5003	*
5004	* @remarks No-long-jump zone!!!
5005	*/
5006	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5007	{
5008	int rc = VINF_SUCCESS;
5009	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5010	{
5011	RTGCUINTREG uVal = 0;
5012	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5013	pMixedCtx->fs.u64Base = uVal;
5014	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5015	}
5016	return rc;
5017	}
5018
5019
5020	/**
5021	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5022	* context.
5023	*
5024	* @returns VBox status code.
5025	* @param pVCpu Pointer to the VMCPU.
5026	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5027	* out-of-sync. Make sure to update the required fields
5028	* before using them.
5029	*
5030	* @remarks No-long-jump zone!!!
5031	*/
5032	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5033	{
5034	int rc = VINF_SUCCESS;
5035	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5036	{
5037	RTGCUINTREG uVal = 0;
5038	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5039	pMixedCtx->gs.u64Base = uVal;
5040	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5041	}
5042	return rc;
5043	}
5044
5045
5046	/**
5047	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5048	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5049	* and TSC_AUX.
5050	*
5051	* @returns VBox status code.
5052	* @param pVCpu Pointer to the VMCPU.
5053	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5054	* out-of-sync. Make sure to update the required fields
5055	* before using them.
5056	*
5057	* @remarks No-long-jump zone!!!
5058	*/
5059	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5060	{
5061	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5062	return VINF_SUCCESS;
5063
5064	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5065	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5066	{
5067	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5068	pMsr += i;
5069	switch (pMsr->u32IndexMSR)
5070	{
5071	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5072	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5073	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5074	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5075	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5076	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5077	default:
5078	{
5079	AssertFailed();
5080	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5081	}
5082	}
5083	}
5084	#endif
5085
5086	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5087	return VINF_SUCCESS;
5088	}
5089
5090
5091	/**
5092	* Saves the guest control registers from the current VMCS into the guest-CPU
5093	* context.
5094	*
5095	* @returns VBox status code.
5096	* @param pVCpu Pointer to the VMCPU.
5097	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5098	* out-of-sync. Make sure to update the required fields
5099	* before using them.
5100	*
5101	* @remarks No-long-jump zone!!!
5102	*/
5103	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5104	{
5105	/* Guest CR0. Guest FPU. */
5106	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5107	AssertRCReturn(rc, rc);
5108
5109	/* Guest CR4. */
5110	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5111	AssertRCReturn(rc, rc);
5112
5113	/* Guest CR2 - updated always during the world-switch or in #PF. */
5114	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5115	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5116	{
5117	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5118	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5119
5120	PVM pVM = pVCpu->CTX_SUFF(pVM);
5121	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5122	\|\| ( pVM->hm.s.fNestedPaging
5123	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5124	{
5125	RTGCUINTREG uVal = 0;
5126	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5127	if (pMixedCtx->cr3 != uVal)
5128	{
5129	CPUMSetGuestCR3(pVCpu, uVal);
5130	if (VMMRZCallRing3IsEnabled(pVCpu))
5131	{
5132	PGMUpdateCR3(pVCpu, uVal);
5133	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5134	}
5135	else
5136	{
5137	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5138	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5139	}
5140	}
5141
5142	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5143	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5144	{
5145	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5146	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5147	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5148	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5149
5150	if (VMMRZCallRing3IsEnabled(pVCpu))
5151	{
5152	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5153	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5154	}
5155	else
5156	{
5157	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5158	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5159	}
5160	}
5161	}
5162
5163	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5164	}
5165
5166	/*
5167	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5168	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5169	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5170	*
5171	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5172	*/
5173	if (VMMRZCallRing3IsEnabled(pVCpu))
5174	{
5175	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5176	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5177
5178	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5179	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5180
5181	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5182	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5183	}
5184
5185	return rc;
5186	}
5187
5188
5189	/**
5190	* Reads a guest segment register from the current VMCS into the guest-CPU
5191	* context.
5192	*
5193	* @returns VBox status code.
5194	* @param pVCpu Pointer to the VMCPU.
5195	* @param idxSel Index of the selector in the VMCS.
5196	* @param idxLimit Index of the segment limit in the VMCS.
5197	* @param idxBase Index of the segment base in the VMCS.
5198	* @param idxAccess Index of the access rights of the segment in the VMCS.
5199	* @param pSelReg Pointer to the segment selector.
5200	*
5201	* @remarks No-long-jump zone!!!
5202	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5203	* as that takes care of whether to read from the VMCS cache or not.
5204	*/
5205	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5206	PCPUMSELREG pSelReg)
5207	{
5208	uint32_t u32Val = 0;
5209	int rc = VMXReadVmcs32(idxSel, &u32Val);
5210	AssertRCReturn(rc, rc);
5211	pSelReg->Sel = (uint16_t)u32Val;
5212	pSelReg->ValidSel = (uint16_t)u32Val;
5213	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5214
5215	rc = VMXReadVmcs32(idxLimit, &u32Val);
5216	AssertRCReturn(rc, rc);
5217	pSelReg->u32Limit = u32Val;
5218
5219	RTGCUINTREG uGCVal = 0;
5220	rc = VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5221	AssertRCReturn(rc, rc);
5222	pSelReg->u64Base = uGCVal;
5223
5224	rc = VMXReadVmcs32(idxAccess, &u32Val);
5225	AssertRCReturn(rc, rc);
5226	pSelReg->Attr.u = u32Val;
5227
5228	/*
5229	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5230	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5231	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5232	*
5233	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5234	*/
5235	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5236	{
5237	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5238	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5239	}
5240	return VINF_SUCCESS;
5241	}
5242
5243	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5244	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5245	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5246	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5247	#else
5248	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5249	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5250	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5251	#endif
5252
5253	/**
5254	* Saves the guest segment registers from the current VMCS into the guest-CPU
5255	* context.
5256	*
5257	* @returns VBox status code.
5258	* @param pVCpu Pointer to the VMCPU.
5259	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5260	* out-of-sync. Make sure to update the required fields
5261	* before using them.
5262	*
5263	* @remarks No-long-jump zone!!!
5264	*/
5265	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5266	{
5267	/* Guest segment registers. */
5268	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5269	{
5270	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5271	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5272	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5273	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5274	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5275	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5276	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5277
5278	/* Restore segment attributes for real-on-v86 mode hack. */
5279	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5280	{
5281	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5282	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5283	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5284	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5285	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5286	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5287	}
5288	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5289	}
5290
5291	return VINF_SUCCESS;
5292	}
5293
5294
5295	/**
5296	* Saves the guest descriptor table registers and task register from the current
5297	* VMCS into the guest-CPU context.
5298	*
5299	* @returns VBox status code.
5300	* @param pVCpu Pointer to the VMCPU.
5301	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5302	* out-of-sync. Make sure to update the required fields
5303	* before using them.
5304	*
5305	* @remarks No-long-jump zone!!!
5306	*/
5307	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5308	{
5309	int rc = VINF_SUCCESS;
5310
5311	/* Guest LDTR. */
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5313	{
5314	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5315	AssertRCReturn(rc, rc);
5316	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5317	}
5318
5319	/* Guest GDTR. */
5320	RTGCUINTREG uGCVal = 0;
5321	uint32_t u32Val = 0;
5322	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5323	{
5324	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal); AssertRCReturn(rc, rc);
5325	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5326	pMixedCtx->gdtr.pGdt = uGCVal;
5327	pMixedCtx->gdtr.cbGdt = u32Val;
5328	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5329	}
5330
5331	/* Guest IDTR. */
5332	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5333	{
5334	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal); AssertRCReturn(rc, rc);
5335	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5336	pMixedCtx->idtr.pIdt = uGCVal;
5337	pMixedCtx->idtr.cbIdt = u32Val;
5338	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5339	}
5340
5341	/* Guest TR. */
5342	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5343	{
5344	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5345	AssertRCReturn(rc, rc);
5346
5347	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5348	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5349	{
5350	rc = VMXLOCAL_READ_SEG(TR, tr);
5351	AssertRCReturn(rc, rc);
5352	}
5353	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5354	}
5355	return rc;
5356	}
5357
5358	#undef VMXLOCAL_READ_SEG
5359
5360
5361	/**
5362	* Saves the guest debug registers from the current VMCS into the guest-CPU
5363	* context.
5364	*
5365	* @returns VBox status code.
5366	* @param pVCpu Pointer to the VMCPU.
5367	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5368	* out-of-sync. Make sure to update the required fields
5369	* before using them.
5370	*
5371	* @remarks No-long-jump zone!!!
5372	*/
5373	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5374	{
5375	int rc = VINF_SUCCESS;
5376	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5377	{
5378	RTGCUINTREG uVal;
5379	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5380	pMixedCtx->dr[7] = uVal;
5381
5382	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5383	}
5384	return rc;
5385	}
5386
5387
5388	/**
5389	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5390	*
5391	* @returns VBox status code.
5392	* @param pVCpu Pointer to the VMCPU.
5393	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5394	* out-of-sync. Make sure to update the required fields
5395	* before using them.
5396	*
5397	* @remarks No-long-jump zone!!!
5398	*/
5399	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5400	{
5401	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5402	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5403	return VINF_SUCCESS;
5404	}
5405
5406
5407	/**
5408	* Saves the entire guest state from the currently active VMCS into the
5409	* guest-CPU context. This essentially VMREADs all guest-data.
5410	*
5411	* @returns VBox status code.
5412	* @param pVCpu Pointer to the VMCPU.
5413	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5414	* out-of-sync. Make sure to update the required fields
5415	* before using them.
5416	*/
5417	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5418	{
5419	Assert(pVCpu);
5420	Assert(pMixedCtx);
5421
5422	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5423	return VINF_SUCCESS;
5424
5425	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5426	there is no real need to. */
5427	if (VMMRZCallRing3IsEnabled(pVCpu))
5428	VMMR0LogFlushDisable(pVCpu);
5429	else
5430	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5431	LogFunc(("\n"));
5432
5433	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5434	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5435
5436	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5437	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5438
5439	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5440	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5441
5442	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5443	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5444
5445	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5446	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5447
5448	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5449	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5450
5451	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5452	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5453
5454	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5455	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5456
5457	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5458	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5459
5460	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5461	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5462
5463	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5465
5466	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5467	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5468
5469	if (VMMRZCallRing3IsEnabled(pVCpu))
5470	VMMR0LogFlushEnable(pVCpu);
5471
5472	return rc;
5473	}
5474
5475
5476	/**
5477	* Check per-VM and per-VCPU force flag actions that require us to go back to
5478	* ring-3 for one reason or another.
5479	*
5480	* @returns VBox status code (information status code included).
5481	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5482	* ring-3.
5483	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5484	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5485	* interrupts)
5486	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5487	* all EMTs to be in ring-3.
5488	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5489	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5490	* to the EM loop.
5491	*
5492	* @param pVM Pointer to the VM.
5493	* @param pVCpu Pointer to the VMCPU.
5494	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5495	* out-of-sync. Make sure to update the required fields
5496	* before using them.
5497	*/
5498	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5499	{
5500	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5501
5502	int rc = VERR_INTERNAL_ERROR_5;
5503	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5504	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5505	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5506	{
5507	/* We need the control registers now, make sure the guest-CPU context is updated. */
5508	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5509	AssertRCReturn(rc, rc);
5510
5511	/* Pending HM CR3 sync. */
5512	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5513	{
5514	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5515	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5516	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5517	}
5518
5519	/* Pending HM PAE PDPEs. */
5520	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5521	{
5522	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5523	AssertRC(rc);
5524	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5525	}
5526
5527	/* Pending PGM C3 sync. */
5528	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5529	{
5530	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5531	if (rc != VINF_SUCCESS)
5532	{
5533	AssertRC(rc);
5534	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5535	return rc;
5536	}
5537	}
5538
5539	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5540	/* -XXX- what was that about single stepping? */
5541	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5542	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5543	{
5544	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5545	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5546	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5547	return rc;
5548	}
5549
5550	/* Pending VM request packets, such as hardware interrupts. */
5551	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5552	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5553	{
5554	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5555	return VINF_EM_PENDING_REQUEST;
5556	}
5557
5558	/* Pending PGM pool flushes. */
5559	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5560	{
5561	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5562	return VINF_PGM_POOL_FLUSH_PENDING;
5563	}
5564
5565	/* Pending DMA requests. */
5566	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5567	{
5568	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5569	return VINF_EM_RAW_TO_R3;
5570	}
5571	}
5572
5573	/* Paranoia. */
5574	Assert(rc != VERR_EM_INTERPRETER);
5575	return VINF_SUCCESS;
5576	}
5577
5578
5579	/**
5580	* Converts any TRPM trap into a pending VMX event. This is typically used when
5581	* entering from ring-3 (not longjmp returns).
5582	*
5583	* @param pVCpu Pointer to the VMCPU.
5584	*/
5585	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5586	{
5587	Assert(TRPMHasTrap(pVCpu));
5588	Assert(!pVCpu->hm.s.Event.fPending);
5589
5590	uint8_t uVector;
5591	TRPMEVENT enmTrpmEvent;
5592	RTGCUINT uErrCode;
5593	RTGCUINTPTR GCPtrFaultAddress;
5594	uint8_t cbInstr;
5595
5596	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5597	AssertRC(rc);
5598
5599	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5600	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5601	if (enmTrpmEvent == TRPM_TRAP)
5602	{
5603	switch (uVector)
5604	{
5605	case X86_XCPT_BP:
5606	case X86_XCPT_OF:
5607	{
5608	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5609	break;
5610	}
5611
5612	case X86_XCPT_PF:
5613	case X86_XCPT_DF:
5614	case X86_XCPT_TS:
5615	case X86_XCPT_NP:
5616	case X86_XCPT_SS:
5617	case X86_XCPT_GP:
5618	case X86_XCPT_AC:
5619	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5620	/* no break! */
5621	default:
5622	{
5623	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5624	break;
5625	}
5626	}
5627	}
5628	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5629	{
5630	if (uVector != X86_XCPT_NMI)
5631	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5632	else
5633	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5634	}
5635	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5636	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5637	else
5638	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5639
5640	rc = TRPMResetTrap(pVCpu);
5641	AssertRC(rc);
5642	Log(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5643	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5644	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5645	}
5646
5647
5648	/**
5649	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5650	* VT-x to execute any instruction.
5651	*
5652	* @param pvCpu Pointer to the VMCPU.
5653	*/
5654	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5655	{
5656	Assert(pVCpu->hm.s.Event.fPending);
5657
5658	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5659	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5660	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5661	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5662
5663	/* If a trap was already pending, we did something wrong! */
5664	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5665
5666	TRPMEVENT enmTrapType;
5667	switch (uVectorType)
5668	{
5669	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5670	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5671	enmTrapType = TRPM_HARDWARE_INT;
5672	break;
5673	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5674	enmTrapType = TRPM_SOFTWARE_INT;
5675	break;
5676	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5677	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5678	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5679	enmTrapType = TRPM_TRAP;
5680	break;
5681	default:
5682	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5683	enmTrapType = TRPM_32BIT_HACK;
5684	break;
5685	}
5686
5687	Log(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5688	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5689	AssertRC(rc);
5690
5691	if (fErrorCodeValid)
5692	TRPMSetErrorCode(pVCpu, uErrorCode);
5693	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5694	&& uVector == X86_XCPT_PF)
5695	{
5696	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5697	}
5698	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5699	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5700	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5701	{
5702	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5703	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5704	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5705	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5706	}
5707	pVCpu->hm.s.Event.fPending = false;
5708	}
5709
5710
5711	/**
5712	* Does the necessary state syncing before doing a longjmp to ring-3.
5713	*
5714	* @param pVM Pointer to the VM.
5715	* @param pVCpu Pointer to the VMCPU.
5716	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5717	* out-of-sync. Make sure to update the required fields
5718	* before using them.
5719	* @param rcExit The reason for exiting to ring-3. Can be
5720	* VINF_VMM_UNKNOWN_RING3_CALL.
5721	*
5722	* @remarks No-long-jmp zone!!!
5723	*/
5724	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5725	{
5726	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5727	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5728
5729	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5730	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5731	AssertRC(rc);
5732
5733	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5734	if (CPUMIsGuestFPUStateActive(pVCpu))
5735	{
5736	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5737	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5738	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5739	}
5740
5741	/* Restore debug registers if necessary and resync on next R0 reentry. */
5742	if (CPUMIsGuestDebugStateActive(pVCpu))
5743	{
5744	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5745	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5746	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5747	}
5748	else if (CPUMIsHyperDebugStateActive(pVCpu))
5749	{
5750	CPUMR0LoadHostDebugState(pVM, pVCpu);
5751	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5752	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5753	}
5754
5755	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5756	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5757	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5758	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5759	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5760	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5761	}
5762
5763
5764	/**
5765	* An action requires us to go back to ring-3. This function does the necessary
5766	* steps before we can safely return to ring-3. This is not the same as longjmps
5767	* to ring-3, this is voluntary.
5768	*
5769	* @param pVM Pointer to the VM.
5770	* @param pVCpu Pointer to the VMCPU.
5771	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5772	* out-of-sync. Make sure to update the required fields
5773	* before using them.
5774	* @param rcExit The reason for exiting to ring-3. Can be
5775	* VINF_VMM_UNKNOWN_RING3_CALL.
5776	*/
5777	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5778	{
5779	Assert(pVM);
5780	Assert(pVCpu);
5781	Assert(pMixedCtx);
5782	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5783
5784	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5785	{
5786	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5787	return;
5788	}
5789	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5790	{
5791	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5792	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5793	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5794	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5795	return;
5796	}
5797
5798	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5799	VMMRZCallRing3Disable(pVCpu);
5800	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5801
5802	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5803	if (pVCpu->hm.s.Event.fPending)
5804	{
5805	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5806	Assert(!pVCpu->hm.s.Event.fPending);
5807	}
5808
5809	/* Sync. the guest state. */
5810	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5811	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5812
5813	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5814	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5815	\| CPUM_CHANGED_LDTR
5816	\| CPUM_CHANGED_GDTR
5817	\| CPUM_CHANGED_IDTR
5818	\| CPUM_CHANGED_TR
5819	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5820
5821	/* On our way back from ring-3 the following needs to be done. */
5822	/** @todo This can change with preemption hooks. */
5823	if (rcExit == VINF_EM_RAW_INTERRUPT)
5824	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5825	else
5826	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5827
5828	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5829	VMMRZCallRing3Enable(pVCpu);
5830	}
5831
5832
5833	/**
5834	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5835	* longjump to ring-3 and possibly get preempted.
5836	*
5837	* @param pVCpu Pointer to the VMCPU.
5838	* @param enmOperation The operation causing the ring-3 longjump.
5839	* @param pvUser The user argument (pointer to the possibly
5840	* out-of-date guest-CPU context).
5841	*
5842	* @remarks Must never be called with @a enmOperation ==
5843	* VMMCALLRING3_VM_R0_ASSERTION.
5844	*/
5845	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5846	{
5847	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5848	Assert(pVCpu);
5849	Assert(pvUser);
5850	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5851	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5852
5853	VMMRZCallRing3Disable(pVCpu);
5854	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5855	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5856	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5857	VMMRZCallRing3Enable(pVCpu);
5858	}
5859
5860
5861	/**
5862	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5863	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5864	*
5865	* @returns VBox status code.
5866	* @param pVCpu Pointer to the VMCPU.
5867	*/
5868	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5869	{
5870	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5871	{
5872	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5873	{
5874	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
5875	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5876	AssertRC(rc);
5877	}
5878	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5879	}
5880
5881
5882	/**
5883	* Injects any pending events into the guest if the guest is in a state to
5884	* receive them.
5885	*
5886	* @returns VBox status code (informational status codes included).
5887	* @param pVCpu Pointer to the VMCPU.
5888	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5889	* out-of-sync. Make sure to update the required fields
5890	* before using them.
5891	*/
5892	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5893	{
5894	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5895	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5896	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5897	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5898
5899	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5900	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5901	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5902	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5903	Assert(!TRPMHasTrap(pVCpu));
5904
5905	int rc = VINF_SUCCESS;
5906	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5907	{
5908	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5909	bool fInject = true;
5910	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5911	{
5912	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5913	AssertRCReturn(rc, rc);
5914	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5915	if ( fBlockInt
5916	\|\| fBlockSti
5917	\|\| fBlockMovSS)
5918	{
5919	fInject = false;
5920	}
5921	}
5922	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5923	&& ( fBlockMovSS
5924	\|\| fBlockSti))
5925	{
5926	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5927	fInject = false;
5928	}
5929
5930	if (fInject)
5931	{
5932	Log(("Injecting pending event\n"));
5933	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5934	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5935	AssertRCReturn(rc, rc);
5936	pVCpu->hm.s.Event.fPending = false;
5937	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5938	}
5939	else
5940	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5941	} /** @todo SMI. SMIs take priority over NMIs. */
5942	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5943	{
5944	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5945	if ( !fBlockMovSS
5946	&& !fBlockSti)
5947	{
5948	Log(("Injecting NMI\n"));
5949	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5950	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5951	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5952	0 /* GCPtrFaultAddress */, &uIntrState);
5953	AssertRCReturn(rc, rc);
5954	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5955	}
5956	else
5957	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5958	}
5959	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5960	{
5961	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5962	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5963	AssertRCReturn(rc, rc);
5964	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5965	if ( !fBlockInt
5966	&& !fBlockSti
5967	&& !fBlockMovSS)
5968	{
5969	uint8_t u8Interrupt;
5970	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5971	if (RT_SUCCESS(rc))
5972	{
5973	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5974	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5975	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5976	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5977	0 /* GCPtrFaultAddress */, &uIntrState);
5978	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5979	}
5980	else
5981	{
5982	/** @todo Does this actually happen? If not turn it into an assertion. */
5983	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5984	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5985	rc = VINF_SUCCESS;
5986	}
5987	}
5988	else
5989	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5990	}
5991
5992	/*
5993	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5994	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5995	*/
5996	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5997	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5998	int rc2 = VINF_SUCCESS;
5999	if ( fBlockSti
6000	\|\| fBlockMovSS)
6001	{
6002	if (!DBGFIsStepping(pVCpu))
6003	{
6004	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6005	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6006	{
6007	/*
6008	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6009	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6010	*/
6011	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6012	AssertRCReturn(rc, rc);
6013	}
6014	}
6015	else
6016	{
6017	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6018	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6019	uIntrState = 0;
6020	}
6021	}
6022
6023	/*
6024	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6025	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6026	*/
6027	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6028	AssertRC(rc2);
6029
6030	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6031	return rc;
6032	}
6033
6034
6035	/**
6036	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6037	*
6038	* @param pVCpu Pointer to the VMCPU.
6039	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6040	* out-of-sync. Make sure to update the required fields
6041	* before using them.
6042	*/
6043	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6044	{
6045	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6046	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6047	}
6048
6049
6050	/**
6051	* Injects a double-fault (#DF) exception into the VM.
6052	*
6053	* @returns VBox status code (informational status code included).
6054	* @param pVCpu Pointer to the VMCPU.
6055	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6056	* out-of-sync. Make sure to update the required fields
6057	* before using them.
6058	*/
6059	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6060	{
6061	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6062	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6063	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6064	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6065	puIntrState);
6066	}
6067
6068
6069	/**
6070	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6071	*
6072	* @param pVCpu Pointer to the VMCPU.
6073	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6074	* out-of-sync. Make sure to update the required fields
6075	* before using them.
6076	*/
6077	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6078	{
6079	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6080	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6081	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6082	}
6083
6084
6085	/**
6086	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6087	*
6088	* @param pVCpu Pointer to the VMCPU.
6089	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6090	* out-of-sync. Make sure to update the required fields
6091	* before using them.
6092	* @param cbInstr The value of RIP that is to be pushed on the guest
6093	* stack.
6094	*/
6095	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6096	{
6097	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6098	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6099	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6100	}
6101
6102
6103	/**
6104	* Injects a general-protection (#GP) fault into the VM.
6105	*
6106	* @returns VBox status code (informational status code included).
6107	* @param pVCpu Pointer to the VMCPU.
6108	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6109	* out-of-sync. Make sure to update the required fields
6110	* before using them.
6111	* @param u32ErrorCode The error code associated with the #GP.
6112	*/
6113	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6114	uint32_t *puIntrState)
6115	{
6116	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6117	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6118	if (fErrorCodeValid)
6119	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6120	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6121	puIntrState);
6122	}
6123
6124
6125	/**
6126	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6127	*
6128	* @param pVCpu Pointer to the VMCPU.
6129	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6130	* out-of-sync. Make sure to update the required fields
6131	* before using them.
6132	* @param uVector The software interrupt vector number.
6133	* @param cbInstr The value of RIP that is to be pushed on the guest
6134	* stack.
6135	*/
6136	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6137	{
6138	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6139	if ( uVector == X86_XCPT_BP
6140	\|\| uVector == X86_XCPT_OF)
6141	{
6142	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6143	}
6144	else
6145	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6146	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6147	}
6148
6149
6150	/**
6151	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6152	* stack.
6153	*
6154	* @returns VBox status code (information status code included).
6155	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6156	* @param pVM Pointer to the VM.
6157	* @param pMixedCtx Pointer to the guest-CPU context.
6158	* @param uValue The value to push to the guest stack.
6159	*/
6160	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6161	{
6162	/*
6163	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6164	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6165	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6166	*/
6167	if (pMixedCtx->sp == 1)
6168	return VINF_EM_RESET;
6169	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6170	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6171	AssertRCReturn(rc, rc);
6172	return rc;
6173	}
6174
6175
6176	/**
6177	* Injects an event into the guest upon VM-entry by updating the relevant fields
6178	* in the VM-entry area in the VMCS.
6179	*
6180	* @returns VBox status code (informational error codes included).
6181	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6182	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6183	*
6184	* @param pVCpu Pointer to the VMCPU.
6185	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6186	* be out-of-sync. Make sure to update the required
6187	* fields before using them.
6188	* @param u64IntrInfo The VM-entry interruption-information field.
6189	* @param cbInstr The VM-entry instruction length in bytes (for
6190	* software interrupts, exceptions and privileged
6191	* software exceptions).
6192	* @param u32ErrCode The VM-entry exception error code.
6193	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6194	* @param puIntrState Pointer to the current guest interruptibility-state.
6195	* This interruptibility-state will be updated if
6196	* necessary. This cannot not be NULL.
6197	*
6198	* @remarks No-long-jump zone!!!
6199	* @remarks Requires CR0!
6200	*/
6201	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6202	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6203	{
6204	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6205	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6206	Assert(puIntrState);
6207	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6208
6209	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6210	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6211
6212	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6213	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6214	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6215
6216	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6217
6218	/* We require CR0 to check if the guest is in real-mode. */
6219	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6220	AssertRCReturn(rc, rc);
6221
6222	/*
6223	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6224	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6225	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6226	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6227	*/
6228	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6229	{
6230	PVM pVM = pVCpu->CTX_SUFF(pVM);
6231	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6232	{
6233	Assert(PDMVmmDevHeapIsEnabled(pVM));
6234	Assert(pVM->hm.s.vmx.pRealModeTSS);
6235
6236	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6237	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6238	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6239	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6240	AssertRCReturn(rc, rc);
6241	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6242
6243	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6244	const size_t cbIdtEntry = 4;
6245	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6246	{
6247	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6248	if (uVector == X86_XCPT_DF)
6249	return VINF_EM_RESET;
6250	else if (uVector == X86_XCPT_GP)
6251	{
6252	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6253	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6254	}
6255
6256	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6257	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6258	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6259	}
6260
6261	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6262	uint16_t uGuestIp = pMixedCtx->ip;
6263	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6264	{
6265	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6266	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6267	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6268	}
6269	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6270	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6271
6272	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6273	uint16_t offIdtEntry = 0;
6274	RTSEL selIdtEntry = 0;
6275	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6276	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6277	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6278	AssertRCReturn(rc, rc);
6279
6280	/* Construct the stack frame for the interrupt/exception handler. */
6281	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6282	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6283	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6284	AssertRCReturn(rc, rc);
6285
6286	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6287	if (rc == VINF_SUCCESS)
6288	{
6289	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6290	pMixedCtx->rip = offIdtEntry;
6291	pMixedCtx->cs.Sel = selIdtEntry;
6292	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6293	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6294	&& uVector == X86_XCPT_PF)
6295	{
6296	pMixedCtx->cr2 = GCPtrFaultAddress;
6297	}
6298	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6299	\| HM_CHANGED_GUEST_RIP
6300	\| HM_CHANGED_GUEST_RFLAGS
6301	\| HM_CHANGED_GUEST_RSP;
6302
6303	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6304	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6305	{
6306	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6307	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6308	Log(("Clearing inhibition due to STI.\n"));
6309	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6310	}
6311	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6312	}
6313	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6314	return rc;
6315	}
6316	else
6317	{
6318	/*
6319	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6320	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6321	*/
6322	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6323	}
6324	}
6325
6326	/* Validate. */
6327	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6328	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6329	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6330
6331	/* Inject. */
6332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6333	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6334	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6335	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6336
6337	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6338	&& uVector == X86_XCPT_PF)
6339	{
6340	pMixedCtx->cr2 = GCPtrFaultAddress;
6341	}
6342	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6343
6344	AssertRCReturn(rc, rc);
6345	return rc;
6346	}
6347
6348
6349	/**
6350	* Enters the VT-x session.
6351	*
6352	* @returns VBox status code.
6353	* @param pVM Pointer to the VM.
6354	* @param pVCpu Pointer to the VMCPU.
6355	* @param pCpu Pointer to the CPU info struct.
6356	*/
6357	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6358	{
6359	AssertPtr(pVM);
6360	AssertPtr(pVCpu);
6361	Assert(pVM->hm.s.vmx.fSupported);
6362	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6363	NOREF(pCpu);
6364
6365	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6366
6367	/* Make sure we're in VMX root mode. */
6368	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6369	if (!(u32HostCR4 & X86_CR4_VMXE))
6370	{
6371	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6372	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6373	}
6374
6375	/* Load the active VMCS as the current one. */
6376	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6377	if (RT_FAILURE(rc))
6378	return rc;
6379
6380	/** @todo this will change with preemption hooks where can can VMRESUME as long
6381	* as we're no preempted. */
6382	pVCpu->hm.s.fResumeVM = false;
6383	return VINF_SUCCESS;
6384	}
6385
6386
6387	/**
6388	* Leaves the VT-x session.
6389	*
6390	* @returns VBox status code.
6391	* @param pVM Pointer to the VM.
6392	* @param pVCpu Pointer to the VMCPU.
6393	* @param pCtx Pointer to the guest-CPU context.
6394	*/
6395	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6396	{
6397	AssertPtr(pVCpu);
6398	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6399	NOREF(pVM);
6400	NOREF(pCtx);
6401
6402	/** @todo this will change with preemption hooks where we only VMCLEAR when
6403	* we are actually going to be preempted, not all the time like we
6404	* currently do. */
6405	/*
6406	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6407	* and mark the VMCS launch-state as "clear".
6408	*/
6409	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6410	return rc;
6411	}
6412
6413
6414	/**
6415	* Saves the host state in the VMCS host-state.
6416	* Sets up the VM-exit MSR-load area.
6417	*
6418	* The CPU state will be loaded from these fields on every successful VM-exit.
6419	*
6420	* @returns VBox status code.
6421	* @param pVM Pointer to the VM.
6422	* @param pVCpu Pointer to the VMCPU.
6423	*
6424	* @remarks No-long-jump zone!!!
6425	*/
6426	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6427	{
6428	AssertPtr(pVM);
6429	AssertPtr(pVCpu);
6430	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6431
6432	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6433
6434	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6435	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6436	return VINF_SUCCESS;
6437
6438	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6439	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6440
6441	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6442	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6443
6444	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6445	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6446
6447	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6448	return rc;
6449	}
6450
6451
6452	/**
6453	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6454	* loaded from these fields on every successful VM-entry.
6455	*
6456	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6457	* Sets up the VM-entry controls.
6458	* Sets up the appropriate VMX non-root function to execute guest code based on
6459	* the guest CPU mode.
6460	*
6461	* @returns VBox status code.
6462	* @param pVM Pointer to the VM.
6463	* @param pVCpu Pointer to the VMCPU.
6464	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6465	* out-of-sync. Make sure to update the required fields
6466	* before using them.
6467	*
6468	* @remarks No-long-jump zone!!!
6469	*/
6470	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6471	{
6472	AssertPtr(pVM);
6473	AssertPtr(pVCpu);
6474	AssertPtr(pMixedCtx);
6475	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6476
6477	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6478
6479	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6480
6481	/* Determine real-on-v86 mode. */
6482	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6483	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6484	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6485	{
6486	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6487	}
6488
6489	/*
6490	* Load the guest-state into the VMCS.
6491	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6492	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6493	*/
6494	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6495	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6496
6497	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6498	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6499
6500	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6501	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6502
6503	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6504	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6505
6506	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6507	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6508	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6509
6510	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6511	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6512
6513	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6514	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6515
6516	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6517	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6518
6519	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6520	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6521
6522	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6523	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6524
6525	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6526	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6527	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6528
6529	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6530	return rc;
6531	}
6532
6533
6534	/**
6535	* Does the preparations before executing guest code in VT-x.
6536	*
6537	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6538	* recompiler. We must be cautious what we do here regarding committing
6539	* guest-state information into the the VMCS assuming we assuredly execute the
6540	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6541	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6542	* that the recompiler can (and should) use them when it resumes guest
6543	* execution. Otherwise such operations must be done when we can no longer
6544	* exit to ring-3.
6545	*
6546	* @returns VBox status code (informational status codes included).
6547	* @retval VINF_SUCCESS if we can proceed with running the guest.
6548	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6549	* into the guest.
6550	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6551	*
6552	* @param pVM Pointer to the VM.
6553	* @param pVCpu Pointer to the VMCPU.
6554	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6555	* out-of-sync. Make sure to update the required fields
6556	* before using them.
6557	* @param pVmxTransient Pointer to the VMX transient structure.
6558	*
6559	* @remarks Called with preemption disabled.
6560	*/
6561	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6562	{
6563	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6564
6565	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6566	PGMRZDynMapFlushAutoSet(pVCpu);
6567	#endif
6568
6569	/* Check force flag actions that might require us to go back to ring-3. */
6570	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6571	if (rc != VINF_SUCCESS)
6572	return rc;
6573
6574	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6575	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6576	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6577	{
6578	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6579	RTGCPHYS GCPhysApicBase;
6580	GCPhysApicBase = pMixedCtx->msrApicBase;
6581	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6582
6583	/* Unalias any existing mapping. */
6584	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6585	AssertRCReturn(rc, rc);
6586
6587	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6588	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6589	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6590	AssertRCReturn(rc, rc);
6591
6592	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6593	}
6594
6595	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6596	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6597	pVmxTransient->uEFlags = ASMIntDisableFlags();
6598	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6599	{
6600	ASMSetFlags(pVmxTransient->uEFlags);
6601	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6602	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6603	return VINF_EM_RAW_INTERRUPT;
6604	}
6605	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6606	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6607	#endif
6608
6609	/*
6610	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6611	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6612	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6613	*/
6614	/** @todo Rework event evaluation and injection to be complete separate. */
6615	if (TRPMHasTrap(pVCpu))
6616	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6617
6618	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6619	AssertRCReturn(rc, rc);
6620	return rc;
6621	}
6622
6623
6624	/**
6625	* Prepares to run guest code in VT-x and we've committed to doing so. This
6626	* means there is no backing out to ring-3 or anywhere else at this
6627	* point.
6628	*
6629	* @param pVM Pointer to the VM.
6630	* @param pVCpu Pointer to the VMCPU.
6631	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6632	* out-of-sync. Make sure to update the required fields
6633	* before using them.
6634	* @param pVmxTransient Pointer to the VMX transient structure.
6635	*
6636	* @remarks Called with preemption disabled.
6637	* @remarks No-long-jump zone!!!
6638	*/
6639	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6640	{
6641	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6642	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6643
6644	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6645	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6646	pVmxTransient->uEFlags = ASMIntDisableFlags();
6647	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6648	#endif
6649
6650	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6651	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6652	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6653	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6654	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6655	#endif
6656	int rc = VINF_SUCCESS;
6657	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6658	{
6659	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6660	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6661	}
6662	else if (pVCpu->hm.s.fContextUseFlags)
6663	{
6664	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6665	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6666	}
6667	AssertRC(rc);
6668	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6669
6670	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6671	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6672	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6673
6674	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6675	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6676	{
6677	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6678	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6679	}
6680
6681	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6682	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6683	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6684
6685	/*
6686	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6687	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6688	*/
6689	if (pVM->hm.s.fTPRPatchingActive)
6690	{
6691	Assert(!CPUMIsGuestInLongMode(pVCpu));
6692
6693	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6694	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6695	AssertRC(rc);
6696
6697	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6698	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6699	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6700	see hmR0VmxLoadGuestApicState(). */
6701	}
6702
6703	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6704	/*
6705	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6706	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6707	*/
6708	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6709	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6710	{
6711	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6712	uint64_t u64HostTscAux = 0;
6713	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6714	AssertRC(rc2);
6715	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6716	}
6717	#endif
6718
6719	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6720	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6721	to start executing. */
6722	}
6723
6724
6725	/**
6726	* Performs some essential restoration of state after running guest code in
6727	* VT-x.
6728	*
6729	* @param pVM Pointer to the VM.
6730	* @param pVCpu Pointer to the VMCPU.
6731	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6732	* out-of-sync. Make sure to update the required fields
6733	* before using them.
6734	* @param pVmxTransient Pointer to the VMX transient structure.
6735	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6736	*
6737	* @remarks Called with interrupts disabled.
6738	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6739	* unconditionally when it is safe to do so.
6740	*/
6741	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6742	{
6743	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6744	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6745
6746	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6747	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6748	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6749	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6750	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6751
6752	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6753	{
6754	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6755	/* Restore host's TSC_AUX. */
6756	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6757	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6758	#endif
6759	/** @todo Find a way to fix hardcoding a guestimate. */
6760	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6761	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6762	}
6763
6764	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6765	Assert(!(ASMGetFlags() & X86_EFL_IF));
6766	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6767
6768	/* Restore the effects of TPR patching if any. */
6769	if (pVM->hm.s.fTPRPatchingActive)
6770	{
6771	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6772	AssertRC(rc);
6773	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6774	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6775	}
6776
6777	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6778	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6779
6780	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6781	uint32_t uExitReason;
6782	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6783	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6784	AssertRC(rc);
6785	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6786	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6787
6788	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6789	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6790
6791	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6792	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6793	{
6794	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6795	return;
6796	}
6797
6798	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6799	{
6800	/* Update the guest interruptibility-state from the VMCS. */
6801	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6802	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6803	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6804	AssertRC(rc);
6805	#endif
6806	/*
6807	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6808	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6809	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6810	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6811	*/
6812	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6813	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6814	{
6815	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6816	AssertRC(rc);
6817	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6818	}
6819	}
6820	}
6821
6822
6823	/**
6824	* Runs the guest code using VT-x.
6825	*
6826	* @returns VBox status code.
6827	* @param pVM Pointer to the VM.
6828	* @param pVCpu Pointer to the VMCPU.
6829	* @param pCtx Pointer to the guest-CPU context.
6830	*
6831	* @remarks Called with preemption disabled.
6832	*/
6833	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6834	{
6835	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6836	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6837
6838	VMXTRANSIENT VmxTransient;
6839	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6840	int rc = VERR_INTERNAL_ERROR_5;
6841	uint32_t cLoops = 0;
6842
6843	for (;; cLoops++)
6844	{
6845	Assert(!HMR0SuspendPending());
6846	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6847	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6848	(unsigned)RTMpCpuId(), cLoops));
6849
6850	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6851	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6852	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6853	if (rc != VINF_SUCCESS)
6854	break;
6855
6856	/*
6857	* No longjmps to ring-3 from this point on!!!
6858	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6859	* This also disables flushing of the R0-logger instance (if any).
6860	*/
6861	VMMRZCallRing3Disable(pVCpu);
6862	VMMRZCallRing3RemoveNotification(pVCpu);
6863	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6864
6865	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6866	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6867
6868	/*
6869	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6870	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6871	*/
6872	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6873	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6874	{
6875	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6876	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6877	return rc;
6878	}
6879
6880	/* Handle the VM-exit. */
6881	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6882	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6883	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6884	HMVMX_START_EXIT_DISPATCH_PROF();
6885	#ifdef HMVMX_USE_FUNCTION_TABLE
6886	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6887	#else
6888	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6889	#endif
6890	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6891	if (rc != VINF_SUCCESS)
6892	break;
6893	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6894	{
6895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6896	rc = VINF_EM_RAW_INTERRUPT;
6897	break;
6898	}
6899	}
6900
6901	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6902	if (rc == VERR_EM_INTERPRETER)
6903	rc = VINF_EM_RAW_EMULATE_INSTR;
6904	else if (rc == VINF_EM_RESET)
6905	rc = VINF_EM_TRIPLE_FAULT;
6906	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6907	return rc;
6908	}
6909
6910
6911	#ifndef HMVMX_USE_FUNCTION_TABLE
6912	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6913	{
6914	int rc;
6915	switch (rcReason)
6916	{
6917	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6918	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6919	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6920	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6921	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6922	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6923	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6924	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6925	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6926	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6927	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6928	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6929	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6930	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6931	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6932	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6933	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6934	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6935	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6936	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6937	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6938	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6939	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6940	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6941	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6942	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6943	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6944	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6945	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6946	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6947	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6948	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6949	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6950
6951	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6952	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6953	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6954	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6955	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6956	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6957	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6958	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6959	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6960
6961	case VMX_EXIT_VMCALL:
6962	case VMX_EXIT_VMCLEAR:
6963	case VMX_EXIT_VMLAUNCH:
6964	case VMX_EXIT_VMPTRLD:
6965	case VMX_EXIT_VMPTRST:
6966	case VMX_EXIT_VMREAD:
6967	case VMX_EXIT_VMRESUME:
6968	case VMX_EXIT_VMWRITE:
6969	case VMX_EXIT_VMXOFF:
6970	case VMX_EXIT_VMXON:
6971	case VMX_EXIT_INVEPT:
6972	case VMX_EXIT_INVVPID:
6973	case VMX_EXIT_VMFUNC:
6974	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6975	break;
6976	default:
6977	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6978	break;
6979	}
6980	return rc;
6981	}
6982	#endif
6983
6984	#ifdef DEBUG
6985	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6986	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6987	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6988	# define VMX_ASSERT_PREEMPT_CPUID() \
6989	do \
6990	{ \
6991	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6992	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6993	} while (0)
6994
6995	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6996	do { \
6997	AssertPtr(pVCpu); \
6998	AssertPtr(pMixedCtx); \
6999	AssertPtr(pVmxTransient); \
7000	Assert(pVmxTransient->fVMEntryFailed == false); \
7001	Assert(ASMIntAreEnabled()); \
7002	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7003	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7004	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7005	(unsigned)pVCpu->idCpu)); \
7006	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7007	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7008	VMX_ASSERT_PREEMPT_CPUID(); \
7009	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7010	} while (0)
7011	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7012	do { \
7013	LogFunc(("\n")); \
7014	} while(0)
7015	#else /* Release builds */
7016	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7017	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7018	#endif
7019
7020
7021	/**
7022	* Advances the guest RIP after reading it from the VMCS.
7023	*
7024	* @returns VBox status code.
7025	* @param pVCpu Pointer to the VMCPU.
7026	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7027	* out-of-sync. Make sure to update the required fields
7028	* before using them.
7029	* @param pVmxTransient Pointer to the VMX transient structure.
7030	*
7031	* @remarks No-long-jump zone!!!
7032	*/
7033	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7034	{
7035	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7036	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7037	AssertRCReturn(rc, rc);
7038
7039	pMixedCtx->rip += pVmxTransient->cbInstr;
7040	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7041	return rc;
7042	}
7043
7044
7045	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7046	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7047	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7048	/**
7049	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7050	*/
7051	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7052	{
7053	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7054	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7055	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7056	Assert(ASMIntAreEnabled());
7057	return VINF_SUCCESS;
7058	#else
7059	return VINF_EM_RAW_INTERRUPT;
7060	#endif
7061	}
7062
7063
7064	/**
7065	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7066	*/
7067	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7068	{
7069	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7070	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7071
7072	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7073	AssertRCReturn(rc, rc);
7074
7075	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7076	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7077	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7078
7079	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7080	{
7081	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7082	return VINF_EM_RAW_INTERRUPT;
7083	}
7084
7085	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7086	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7087	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7088	{
7089	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7090	return VINF_SUCCESS;
7091	}
7092	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7093	{
7094	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7095	return rc;
7096	}
7097
7098	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7099	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7100	switch (uIntrType)
7101	{
7102	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7103	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7104	/* no break */
7105	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7106	{
7107	switch (uVector)
7108	{
7109	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7110	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7111	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7112	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7113	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7114	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7115	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7116	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7117	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7118	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7119	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7120	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7121	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7122	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7123	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7124	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7125	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7126	#endif
7127	default:
7128	{
7129	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7130	AssertRCReturn(rc, rc);
7131
7132	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7133	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7134	{
7135	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7136	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7137	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7138	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7139	AssertRCReturn(rc, rc);
7140	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7141	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7142	0 /* GCPtrFaultAddress */);
7143	AssertRCReturn(rc, rc);
7144	}
7145	else
7146	{
7147	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7148	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7149	}
7150	break;
7151	}
7152	}
7153	break;
7154	}
7155
7156	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7157	default:
7158	{
7159	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7160	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7161	break;
7162	}
7163	}
7164	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7165	return rc;
7166	}
7167
7168
7169	/**
7170	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7171	*/
7172	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7173	{
7174	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7175
7176	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7177	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7178	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7179	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7180	AssertRCReturn(rc, rc);
7181
7182	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7183	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7184	return VINF_SUCCESS;
7185	}
7186
7187
7188	/**
7189	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7190	*/
7191	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7192	{
7193	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7194	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7195	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7196	}
7197
7198
7199	/**
7200	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7201	*/
7202	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7203	{
7204	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7205	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7206	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7207	}
7208
7209
7210	/**
7211	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7212	*/
7213	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7214	{
7215	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7217	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7218	}
7219
7220
7221	/**
7222	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7223	*/
7224	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7225	{
7226	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7227	PVM pVM = pVCpu->CTX_SUFF(pVM);
7228	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7229	if (RT_LIKELY(rc == VINF_SUCCESS))
7230	{
7231	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7232	Assert(pVmxTransient->cbInstr == 2);
7233	}
7234	else
7235	{
7236	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7237	rc = VERR_EM_INTERPRETER;
7238	}
7239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7240	return rc;
7241	}
7242
7243
7244	/**
7245	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7246	*/
7247	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7248	{
7249	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7250	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7251	AssertRCReturn(rc, rc);
7252
7253	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7254	return VINF_EM_RAW_EMULATE_INSTR;
7255
7256	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7257	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7258	}
7259
7260
7261	/**
7262	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7263	*/
7264	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7265	{
7266	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7267	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7268	AssertRCReturn(rc, rc);
7269
7270	PVM pVM = pVCpu->CTX_SUFF(pVM);
7271	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7272	if (RT_LIKELY(rc == VINF_SUCCESS))
7273	{
7274	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7275	Assert(pVmxTransient->cbInstr == 2);
7276	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7277	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7278	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7279	}
7280	else
7281	{
7282	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7283	rc = VERR_EM_INTERPRETER;
7284	}
7285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7286	return rc;
7287	}
7288
7289
7290	/**
7291	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7292	*/
7293	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7294	{
7295	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7296	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7297	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7298	AssertRCReturn(rc, rc);
7299
7300	PVM pVM = pVCpu->CTX_SUFF(pVM);
7301	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7302	if (RT_LIKELY(rc == VINF_SUCCESS))
7303	{
7304	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7305	Assert(pVmxTransient->cbInstr == 3);
7306	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7307	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7308	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7309	}
7310	else
7311	{
7312	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7313	rc = VERR_EM_INTERPRETER;
7314	}
7315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7316	return rc;
7317	}
7318
7319
7320	/**
7321	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7322	*/
7323	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7324	{
7325	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7326	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7327	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7328	AssertRCReturn(rc, rc);
7329
7330	PVM pVM = pVCpu->CTX_SUFF(pVM);
7331	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7332	if (RT_LIKELY(rc == VINF_SUCCESS))
7333	{
7334	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7335	Assert(pVmxTransient->cbInstr == 2);
7336	}
7337	else
7338	{
7339	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7340	rc = VERR_EM_INTERPRETER;
7341	}
7342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7343	return rc;
7344	}
7345
7346
7347	/**
7348	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7349	*/
7350	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7351	{
7352	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7353	PVM pVM = pVCpu->CTX_SUFF(pVM);
7354	Assert(!pVM->hm.s.fNestedPaging);
7355
7356	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7357	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7358	AssertRCReturn(rc, rc);
7359
7360	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7361	rc = VBOXSTRICTRC_VAL(rc2);
7362	if (RT_LIKELY(rc == VINF_SUCCESS))
7363	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7364	else
7365	{
7366	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7367	pVmxTransient->uExitQualification, rc));
7368	}
7369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7370	return rc;
7371	}
7372
7373
7374	/**
7375	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7376	*/
7377	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7378	{
7379	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7380	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7381	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7382	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7383	AssertRCReturn(rc, rc);
7384
7385	PVM pVM = pVCpu->CTX_SUFF(pVM);
7386	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7387	if (RT_LIKELY(rc == VINF_SUCCESS))
7388	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7389	else
7390	{
7391	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7392	rc = VERR_EM_INTERPRETER;
7393	}
7394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7395	return rc;
7396	}
7397
7398
7399	/**
7400	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7401	*/
7402	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7403	{
7404	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7405	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7406	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7407	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7408	AssertRCReturn(rc, rc);
7409
7410	PVM pVM = pVCpu->CTX_SUFF(pVM);
7411	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7412	rc = VBOXSTRICTRC_VAL(rc2);
7413	if (RT_LIKELY( rc == VINF_SUCCESS
7414	\|\| rc == VINF_EM_HALT))
7415	{
7416	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7417	AssertRCReturn(rc3, rc3);
7418
7419	if ( rc == VINF_EM_HALT
7420	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7421	{
7422	rc = VINF_SUCCESS;
7423	}
7424	}
7425	else
7426	{
7427	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7428	rc = VERR_EM_INTERPRETER;
7429	}
7430	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7431	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7433	return rc;
7434	}
7435
7436
7437	/**
7438	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7439	*/
7440	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7441	{
7442	/*
7443	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7444	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7445	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7446	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7447	*/
7448	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7449	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7450	}
7451
7452
7453	/**
7454	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7455	*/
7456	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7457	{
7458	/*
7459	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7460	* root operation. If we get there there is something funny going on.
7461	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7462	*/
7463	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7464	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7465	}
7466
7467
7468	/**
7469	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7470	*/
7471	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7472	{
7473	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7474	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7475	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7476	}
7477
7478
7479	/**
7480	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7481	*/
7482	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7483	{
7484	/*
7485	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7486	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7487	* See Intel spec. 25.3 "Other Causes of VM-exits".
7488	*/
7489	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7490	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7491	}
7492
7493
7494	/**
7495	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7496	* VM-exit.
7497	*/
7498	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7499	{
7500	/*
7501	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7502	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7503	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7504	*/
7505	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7506	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7507	}
7508
7509
7510	/**
7511	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7512	* VM-exit.
7513	*/
7514	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7515	{
7516	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7517	return VINF_EM_RESET;
7518	}
7519
7520
7521	/**
7522	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7523	*/
7524	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7525	{
7526	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7527	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7528	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7529	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7530	AssertRCReturn(rc, rc);
7531
7532	pMixedCtx->rip++;
7533	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7534	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7535	rc = VINF_SUCCESS;
7536	else
7537	rc = VINF_EM_HALT;
7538
7539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7540	return rc;
7541	}
7542
7543
7544	/**
7545	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7546	*/
7547	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7548	{
7549	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7550	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7551	return VINF_SUCCESS;
7552	}
7553
7554
7555	/**
7556	* VM-exit handler for expiry of the VMX preemption timer.
7557	*/
7558	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7559	{
7560	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7561
7562	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7563	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7564
7565	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7566	PVM pVM = pVCpu->CTX_SUFF(pVM);
7567	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7569	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7570	}
7571
7572
7573	/**
7574	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7575	*/
7576	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7577	{
7578	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7579	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7580	/** @todo check if XSETBV is supported by the recompiler. */
7581	return VERR_EM_INTERPRETER;
7582	}
7583
7584
7585	/**
7586	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7587	*/
7588	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7589	{
7590	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7591	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7592	/** @todo implement EMInterpretInvpcid() */
7593	return VERR_EM_INTERPRETER;
7594	}
7595
7596
7597	/**
7598	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7599	* Error VM-exit.
7600	*/
7601	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7602	{
7603	uint32_t uIntrState;
7604	HMVMXHCUINTREG uHCReg;
7605	uint64_t u64Val;
7606	uint32_t u32Val;
7607
7608	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7609	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7610	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7611	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7612	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7613	AssertRCReturn(rc, rc);
7614
7615	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7616	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7617	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7618	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7619
7620	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7621	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7622	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7623	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7624	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7625	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7626	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7627	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7628	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7629	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7630	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7631	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7632
7633	PVM pVM = pVCpu->CTX_SUFF(pVM);
7634	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7635
7636	return VERR_VMX_INVALID_GUEST_STATE;
7637	}
7638
7639
7640	/**
7641	* VM-exit handler for VM-entry failure due to an MSR-load
7642	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7643	*/
7644	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7645	{
7646	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7647	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7648	}
7649
7650
7651	/**
7652	* VM-exit handler for VM-entry failure due to a machine-check event
7653	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7654	*/
7655	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7656	{
7657	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7658	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7659	}
7660
7661
7662	/**
7663	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7664	* theory.
7665	*/
7666	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7667	{
7668	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7669	return VERR_VMX_UNDEFINED_EXIT_CODE;
7670	}
7671
7672
7673	/**
7674	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7675	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7676	* Conditional VM-exit.
7677	*/
7678	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7679	{
7680	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7681	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7683	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7684	return VERR_EM_INTERPRETER;
7685	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7686	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7687	}
7688
7689
7690	/**
7691	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7692	*/
7693	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7694	{
7695	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7696	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7697	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7698	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7699	return VERR_EM_INTERPRETER;
7700	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7701	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7702	}
7703
7704
7705	/**
7706	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7707	*/
7708	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7709	{
7710	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7711	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7712	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7713	AssertRCReturn(rc, rc);
7714	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7715	AssertRCReturn(rc, rc);
7716	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7717	AssertRCReturn(rc, rc);
7718
7719	PVM pVM = pVCpu->CTX_SUFF(pVM);
7720	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7721	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7722	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7724
7725	if (RT_LIKELY(rc == VINF_SUCCESS))
7726	{
7727	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7728	Assert(pVmxTransient->cbInstr == 2);
7729	}
7730	return rc;
7731	}
7732
7733
7734	/**
7735	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7736	*/
7737	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7738	{
7739	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7740	PVM pVM = pVCpu->CTX_SUFF(pVM);
7741	int rc = VINF_SUCCESS;
7742
7743	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7744	if ( pVM->hm.s.fTPRPatchingActive
7745	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7746	{
7747	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7748	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7749	{
7750	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7751	AssertRC(rc);
7752	}
7753
7754	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7755	Assert(pVmxTransient->cbInstr == 2);
7756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7757	return VINF_SUCCESS;
7758	}
7759
7760	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7761	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7762	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7763	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7764	AssertRCReturn(rc, rc);
7765	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7766
7767	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7768	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7769	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7770
7771	if (RT_LIKELY(rc == VINF_SUCCESS))
7772	{
7773	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7774
7775	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7776	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7777	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7778	{
7779	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7780	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7781	}
7782	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7783	{
7784	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7785	AssertRCReturn(rc, rc);
7786	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7787	}
7788	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7789	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7790
7791	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7792	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)))
7793	{
7794	switch (pMixedCtx->ecx)
7795	{
7796	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7797	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7798	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7799	case MSR_K8_FS_BASE: /* no break */
7800	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7801	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7802	}
7803	}
7804	#ifdef VBOX_STRICT
7805	else
7806	{
7807	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7808	switch (pMixedCtx->ecx)
7809	{
7810	case MSR_IA32_SYSENTER_CS:
7811	case MSR_IA32_SYSENTER_EIP:
7812	case MSR_IA32_SYSENTER_ESP:
7813	case MSR_K8_FS_BASE:
7814	case MSR_K8_GS_BASE:
7815	{
7816	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7817	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7818	}
7819
7820	case MSR_K8_LSTAR:
7821	case MSR_K6_STAR:
7822	case MSR_K8_SF_MASK:
7823	case MSR_K8_TSC_AUX:
7824	case MSR_K8_KERNEL_GS_BASE:
7825	{
7826	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7827	pMixedCtx->ecx));
7828	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7829	}
7830	}
7831	}
7832	#endif /* VBOX_STRICT */
7833	}
7834	return rc;
7835	}
7836
7837
7838	/**
7839	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7840	*/
7841	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7842	{
7843	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7844	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7846	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7847	return VERR_EM_INTERPRETER;
7848	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7849	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7850	}
7851
7852
7853	/**
7854	* VM-exit handler for when the TPR value is lowered below the specified
7855	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7856	*/
7857	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7858	{
7859	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7860	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
7861
7862	/*
7863	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7864	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7865	* resume guest execution.
7866	*/
7867	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7869	return VINF_SUCCESS;
7870	}
7871
7872
7873	/**
7874	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7875	* VM-exit.
7876	*
7877	* @retval VINF_SUCCESS when guest execution can continue.
7878	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7879	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7880	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7881	* recompiler.
7882	*/
7883	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7884	{
7885	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7886	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7887	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7888	AssertRCReturn(rc, rc);
7889
7890	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7891	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7892	PVM pVM = pVCpu->CTX_SUFF(pVM);
7893	switch (uAccessType)
7894	{
7895	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7896	{
7897	#if 0
7898	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7899	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7900	#else
7901	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7902	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7903	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7904	#endif
7905	AssertRCReturn(rc, rc);
7906
7907	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7908	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7909	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7910	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7911
7912	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7913	{
7914	case 0: /* CR0 */
7915	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7916	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7917	break;
7918	case 2: /* C2 **/
7919	/* Nothing to do here, CR2 it's not part of the VMCS. */
7920	break;
7921	case 3: /* CR3 */
7922	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7923	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7924	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7925	break;
7926	case 4: /* CR4 */
7927	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7928	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7929	break;
7930	case 8: /* CR8 */
7931	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7932	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7933	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7934	break;
7935	default:
7936	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7937	break;
7938	}
7939
7940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7941	break;
7942	}
7943
7944	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7945	{
7946	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7947	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7948	AssertRCReturn(rc, rc);
7949	Assert( !pVM->hm.s.fNestedPaging
7950	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7951	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7952
7953	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7954	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7955	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7956
7957	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7958	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7959	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7960	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7962	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7963	break;
7964	}
7965
7966	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7967	{
7968	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7969	AssertRCReturn(rc, rc);
7970	rc = EMInterpretCLTS(pVM, pVCpu);
7971	AssertRCReturn(rc, rc);
7972	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7974	Log(("CRX CLTS write rc=%d\n", rc));
7975	break;
7976	}
7977
7978	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7979	{
7980	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7981	AssertRCReturn(rc, rc);
7982	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7983	if (RT_LIKELY(rc == VINF_SUCCESS))
7984	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7985	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7986	Log(("CRX LMSW write rc=%d\n", rc));
7987	break;
7988	}
7989
7990	default:
7991	{
7992	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7993	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7994	}
7995	}
7996
7997	/* Validate possible error codes. */
7998	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7999	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8000	if (RT_SUCCESS(rc))
8001	{
8002	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8003	AssertRCReturn(rc2, rc2);
8004	}
8005
8006	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8007	return rc;
8008	}
8009
8010
8011	/**
8012	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8013	* VM-exit.
8014	*/
8015	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8016	{
8017	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8018	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8019
8020	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8021	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8022	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8023	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8024	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8025	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8026	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8027	AssertRCReturn(rc, rc);
8028	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8029
8030	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8031	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8032	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8033	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8034	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8035	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8036	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8037
8038	/* I/O operation lookup arrays. */
8039	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8040	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8041
8042	const uint32_t cbSize = s_aIOSize[uIOWidth];
8043	const uint32_t cbInstr = pVmxTransient->cbInstr;
8044	PVM pVM = pVCpu->CTX_SUFF(pVM);
8045	if (fIOString)
8046	{
8047	/* INS/OUTS - I/O String instruction. */
8048	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8049	/** @todo for now manually disassemble later optimize by getting the fields from
8050	* the VMCS. */
8051	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8052	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8053	* segment prefix info. */
8054	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8055	if (RT_SUCCESS(rc))
8056	{
8057	if (fIOWrite)
8058	{
8059	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8060	(DISCPUMODE)pDis->uAddrMode, cbSize);
8061	rc = VBOXSTRICTRC_VAL(rc2);
8062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8063	}
8064	else
8065	{
8066	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8067	(DISCPUMODE)pDis->uAddrMode, cbSize);
8068	rc = VBOXSTRICTRC_VAL(rc2);
8069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8070	}
8071	}
8072	else
8073	{
8074	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8075	rc = VINF_EM_RAW_EMULATE_INSTR;
8076	}
8077	}
8078	else
8079	{
8080	/* IN/OUT - I/O instruction. */
8081	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8082	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8083	if (fIOWrite)
8084	{
8085	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8086	rc = VBOXSTRICTRC_VAL(rc2);
8087	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8088	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8090	}
8091	else
8092	{
8093	uint32_t u32Result = 0;
8094	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8095	rc = VBOXSTRICTRC_VAL(rc2);
8096	if (IOM_SUCCESS(rc))
8097	{
8098	/* Save result of I/O IN instr. in AL/AX/EAX. */
8099	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8100	}
8101	else if (rc == VINF_IOM_R3_IOPORT_READ)
8102	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8104	}
8105	}
8106
8107	if (IOM_SUCCESS(rc))
8108	{
8109	pMixedCtx->rip += cbInstr;
8110	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8111	if (RT_LIKELY(rc == VINF_SUCCESS))
8112	{
8113	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8114	AssertRCReturn(rc, rc);
8115
8116	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8117	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8118	{
8119	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8120	for (unsigned i = 0; i < 4; i++)
8121	{
8122	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8123	if ( ( uIOPort >= pMixedCtx->dr[i]
8124	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8125	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8126	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8127	{
8128	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8129	uint64_t uDR6 = ASMGetDR6();
8130
8131	/* Clear all breakpoint status flags and set the one we just hit. */
8132	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8133	uDR6 \|= (uint64_t)RT_BIT(i);
8134
8135	/*
8136	* Note: AMD64 Architecture Programmer's Manual 13.1:
8137	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8138	* be cleared by software after the contents have been read.
8139	*/
8140	ASMSetDR6(uDR6);
8141
8142	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8143	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8144
8145	/* Paranoia. */
8146	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8147	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8148	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8149
8150	/* Resync DR7 */
8151	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8152	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8153
8154	/* Set #DB to be injected into the VM and continue guest execution. */
8155	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8156	break;
8157	}
8158	}
8159	}
8160	}
8161	}
8162
8163	#ifdef DEBUG
8164	if (rc == VINF_IOM_R3_IOPORT_READ)
8165	Assert(!fIOWrite);
8166	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8167	Assert(fIOWrite);
8168	else
8169	{
8170	AssertMsg( RT_FAILURE(rc)
8171	\|\| rc == VINF_SUCCESS
8172	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8173	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8174	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8175	}
8176	#endif
8177
8178	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8179	return rc;
8180	}
8181
8182
8183	/**
8184	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8185	* VM-exit.
8186	*/
8187	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8188	{
8189	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8190
8191	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8192	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8193	AssertRCReturn(rc, rc);
8194	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8195	{
8196	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8197	AssertRCReturn(rc, rc);
8198	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8199	{
8200	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8201	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8202	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8203	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8204	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8205	{
8206	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8207	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8208
8209	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8210	Assert(!pVCpu->hm.s.Event.fPending);
8211	pVCpu->hm.s.Event.fPending = true;
8212	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8213	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8214	AssertRCReturn(rc, rc);
8215	if (fErrorCodeValid)
8216	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8217	else
8218	pVCpu->hm.s.Event.u32ErrCode = 0;
8219	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8220	&& uVector == X86_XCPT_PF)
8221	{
8222	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8223	}
8224	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8225	}
8226	}
8227	}
8228	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8229	* emulation. */
8230	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8231	return VERR_EM_INTERPRETER;
8232	}
8233
8234
8235	/**
8236	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8237	*/
8238	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8239	{
8240	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8241	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8242	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8243	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8244	AssertRCReturn(rc, rc);
8245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8246	return VINF_EM_DBG_STOP;
8247	}
8248
8249
8250	/**
8251	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8252	*/
8253	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8254	{
8255	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8256
8257	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8258	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8259	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8260	return VINF_SUCCESS;
8261	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8262	return rc;
8263
8264	#if 0
8265	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8266	* just sync the whole thing. */
8267	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8268	#else
8269	/* Aggressive state sync. for now. */
8270	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8271	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8272	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8273	#endif
8274	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8275	AssertRCReturn(rc, rc);
8276
8277	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8278	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8279	switch (uAccessType)
8280	{
8281	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8282	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8283	{
8284	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8285	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8286	{
8287	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8288	}
8289
8290	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8291	GCPhys &= PAGE_BASE_GC_MASK;
8292	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8293	PVM pVM = pVCpu->CTX_SUFF(pVM);
8294	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8295	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8296
8297	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8298	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8299	CPUMCTX2CORE(pMixedCtx), GCPhys);
8300	rc = VBOXSTRICTRC_VAL(rc2);
8301	Log(("ApicAccess rc=%d\n", rc));
8302	if ( rc == VINF_SUCCESS
8303	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8304	\|\| rc == VERR_PAGE_NOT_PRESENT)
8305	{
8306	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8307	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8308	rc = VINF_SUCCESS;
8309	}
8310	break;
8311	}
8312
8313	default:
8314	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8315	rc = VINF_EM_RAW_EMULATE_INSTR;
8316	break;
8317	}
8318
8319	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8320	return rc;
8321	}
8322
8323
8324	/**
8325	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8326	* VM-exit.
8327	*/
8328	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8329	{
8330	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8331
8332	/* We should -not- get this VM-exit if the guest is debugging. */
8333	if (CPUMIsGuestDebugStateActive(pVCpu))
8334	{
8335	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8336	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8337	}
8338
8339	int rc = VERR_INTERNAL_ERROR_5;
8340	if ( !DBGFIsStepping(pVCpu)
8341	&& !CPUMIsHyperDebugStateActive(pVCpu))
8342	{
8343	/* Don't intercept MOV DRx. */
8344	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8345	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8346	AssertRCReturn(rc, rc);
8347
8348	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8349	PVM pVM = pVCpu->CTX_SUFF(pVM);
8350	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8351	AssertRC(rc);
8352	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8353
8354	#ifdef VBOX_WITH_STATISTICS
8355	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8356	AssertRCReturn(rc, rc);
8357	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8358	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8359	else
8360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8361	#endif
8362	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8363	return VINF_SUCCESS;
8364	}
8365
8366	/*
8367	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8368	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8369	*/
8370	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8371	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8372	AssertRCReturn(rc, rc);
8373
8374	PVM pVM = pVCpu->CTX_SUFF(pVM);
8375	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8376	{
8377	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8378	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8379	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8380	if (RT_SUCCESS(rc))
8381	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8383	}
8384	else
8385	{
8386	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8387	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8388	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8390	}
8391
8392	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8393	if (RT_SUCCESS(rc))
8394	{
8395	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8396	AssertRCReturn(rc2, rc2);
8397	}
8398	return rc;
8399	}
8400
8401
8402	/**
8403	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8404	* Conditional VM-exit.
8405	*/
8406	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8407	{
8408	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8409	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8410
8411	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8412	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8413	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8414	return VINF_SUCCESS;
8415	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8416	return rc;
8417
8418	RTGCPHYS GCPhys = 0;
8419	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8420
8421	#if 0
8422	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8423	#else
8424	/* Aggressive state sync. for now. */
8425	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8426	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8427	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8428	#endif
8429	AssertRCReturn(rc, rc);
8430
8431	/*
8432	* If we succeed, resume guest execution.
8433	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8434	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8435	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8436	* weird case. See @bugref{6043}.
8437	*/
8438	PVM pVM = pVCpu->CTX_SUFF(pVM);
8439	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8440	rc = VBOXSTRICTRC_VAL(rc2);
8441	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8442	if ( rc == VINF_SUCCESS
8443	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8444	\|\| rc == VERR_PAGE_NOT_PRESENT)
8445	{
8446	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8447	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8448	return VINF_SUCCESS;
8449	}
8450	return rc;
8451	}
8452
8453
8454	/**
8455	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8456	* VM-exit.
8457	*/
8458	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8459	{
8460	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8461	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8462
8463	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8464	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8465	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8466	return VINF_SUCCESS;
8467	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8468	return rc;
8469
8470	RTGCPHYS GCPhys = 0;
8471	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8472	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8473	#if 0
8474	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8475	#else
8476	/* Aggressive state sync. for now. */
8477	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8478	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8479	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8480	#endif
8481	AssertRCReturn(rc, rc);
8482
8483	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8484	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8485
8486	RTGCUINT uErrorCode = 0;
8487	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8488	uErrorCode \|= X86_TRAP_PF_ID;
8489	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8490	uErrorCode \|= X86_TRAP_PF_RW;
8491	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8492	uErrorCode \|= X86_TRAP_PF_P;
8493
8494	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8495
8496	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8497	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8498
8499	/* Handle the pagefault trap for the nested shadow table. */
8500	PVM pVM = pVCpu->CTX_SUFF(pVM);
8501	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8502	TRPMResetTrap(pVCpu);
8503
8504	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8505	if ( rc == VINF_SUCCESS
8506	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8507	\|\| rc == VERR_PAGE_NOT_PRESENT)
8508	{
8509	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8511	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8512	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8513	return VINF_SUCCESS;
8514	}
8515
8516	Log(("EPT return to ring-3 rc=%d\n"));
8517	return rc;
8518	}
8519
8520
8521	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8522	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8523	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8524	/**
8525	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8526	*/
8527	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8528	{
8529	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8530	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8531
8532	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8533	AssertRCReturn(rc, rc);
8534
8535	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8536	{
8537	/* Old-style FPU error reporting needs some extra work. */
8538	/** @todo don't fall back to the recompiler, but do it manually. */
8539	return VERR_EM_INTERPRETER;
8540	}
8541	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8542	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8543	return rc;
8544	}
8545
8546
8547	/**
8548	* VM-exit exception handler for #BP (Breakpoint exception).
8549	*/
8550	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8551	{
8552	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8553	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8554
8555	/** @todo Try optimize this by not saving the entire guest state unless
8556	* really needed. */
8557	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8558	AssertRCReturn(rc, rc);
8559
8560	PVM pVM = pVCpu->CTX_SUFF(pVM);
8561	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8562	if (rc == VINF_EM_RAW_GUEST_TRAP)
8563	{
8564	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8565	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8566	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8567	AssertRCReturn(rc, rc);
8568
8569	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8570	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8571	}
8572
8573	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8574	return rc;
8575	}
8576
8577
8578	/**
8579	* VM-exit exception handler for #DB (Debug exception).
8580	*/
8581	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8582	{
8583	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8584	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8585
8586	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8587	AssertRCReturn(rc, rc);
8588
8589	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8590	uint64_t uDR6 = X86_DR6_INIT_VAL;
8591	uDR6 \|= (pVmxTransient->uExitQualification
8592	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8593	PVM pVM = pVCpu->CTX_SUFF(pVM);
8594	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8595	if (rc == VINF_EM_RAW_GUEST_TRAP)
8596	{
8597	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8598	pMixedCtx->dr[6] = uDR6;
8599
8600	if (CPUMIsGuestDebugStateActive(pVCpu))
8601	ASMSetDR6(pMixedCtx->dr[6]);
8602
8603	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8604
8605	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8606	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8607
8608	/* Paranoia. */
8609	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8610	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8611	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8612
8613	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8614	AssertRCReturn(rc,rc);
8615
8616	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8617	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8618	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8619	AssertRCReturn(rc2, rc2);
8620	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8621	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8622	rc = VINF_SUCCESS;
8623	}
8624
8625	return rc;
8626	}
8627
8628
8629	/**
8630	* VM-exit exception handler for #NM (Device-not-available exception: floating
8631	* point exception).
8632	*/
8633	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8634	{
8635	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8636
8637	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8638	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8639	#endif
8640
8641	/* We require CR0 and EFER. EFER is always up-to-date. */
8642	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8643	AssertRCReturn(rc, rc);
8644
8645	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8646	PVM pVM = pVCpu->CTX_SUFF(pVM);
8647	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8648	if (rc == VINF_SUCCESS)
8649	{
8650	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8651	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8653	return VINF_SUCCESS;
8654	}
8655
8656	/* Forward #NM to the guest. */
8657	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8658	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8659	AssertRCReturn(rc, rc);
8660	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8661	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8663	return rc;
8664	}
8665
8666
8667	/**
8668	* VM-exit exception handler for #GP (General-protection exception).
8669	*
8670	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8671	*/
8672	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8673	{
8674	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8676
8677	int rc = VERR_INTERNAL_ERROR_5;
8678	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8679	{
8680	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8681	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8682	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8683	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8684	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8685	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8686	AssertRCReturn(rc, rc);
8687	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8688	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8689	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8690	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8691	return rc;
8692	#else
8693	/* We don't intercept #GP. */
8694	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8695	return VERR_VMX_UNEXPECTED_EXCEPTION;
8696	#endif
8697	}
8698
8699	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8700	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8701
8702	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8703	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8704	AssertRCReturn(rc, rc);
8705
8706	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8707	uint32_t cbOp = 0;
8708	PVM pVM = pVCpu->CTX_SUFF(pVM);
8709	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8710	if (RT_SUCCESS(rc))
8711	{
8712	rc = VINF_SUCCESS;
8713	Assert(cbOp == pDis->cbInstr);
8714	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8715	switch (pDis->pCurInstr->uOpcode)
8716	{
8717	case OP_CLI:
8718	pMixedCtx->eflags.Bits.u1IF = 0;
8719	pMixedCtx->rip += pDis->cbInstr;
8720	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8721	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8722	break;
8723
8724	case OP_STI:
8725	pMixedCtx->eflags.Bits.u1IF = 1;
8726	pMixedCtx->rip += pDis->cbInstr;
8727	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8728	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8729	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8730	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8731	break;
8732
8733	case OP_HLT:
8734	rc = VINF_EM_HALT;
8735	pMixedCtx->rip += pDis->cbInstr;
8736	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8738	break;
8739
8740	case OP_POPF:
8741	{
8742	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8743	uint32_t cbParm = 0;
8744	uint32_t uMask = 0;
8745	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8746	{
8747	cbParm = 4;
8748	uMask = 0xffffffff;
8749	}
8750	else
8751	{
8752	cbParm = 2;
8753	uMask = 0xffff;
8754	}
8755
8756	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8757	RTGCPTR GCPtrStack = 0;
8758	X86EFLAGS uEflags;
8759	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8760	&GCPtrStack);
8761	if (RT_SUCCESS(rc))
8762	{
8763	Assert(sizeof(uEflags.u32) >= cbParm);
8764	uEflags.u32 = 0;
8765	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8766	}
8767	if (RT_FAILURE(rc))
8768	{
8769	rc = VERR_EM_INTERPRETER;
8770	break;
8771	}
8772	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8773	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8774	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8775	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8776	pMixedCtx->eflags.Bits.u1RF = 0;
8777	pMixedCtx->esp += cbParm;
8778	pMixedCtx->esp &= uMask;
8779	pMixedCtx->rip += pDis->cbInstr;
8780	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8782	break;
8783	}
8784
8785	case OP_PUSHF:
8786	{
8787	uint32_t cbParm = 0;
8788	uint32_t uMask = 0;
8789	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8790	{
8791	cbParm = 4;
8792	uMask = 0xffffffff;
8793	}
8794	else
8795	{
8796	cbParm = 2;
8797	uMask = 0xffff;
8798	}
8799
8800	/* Get the stack pointer & push the contents of eflags onto the stack. */
8801	RTGCPTR GCPtrStack = 0;
8802	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8803	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8804	if (RT_FAILURE(rc))
8805	{
8806	rc = VERR_EM_INTERPRETER;
8807	break;
8808	}
8809	X86EFLAGS uEflags;
8810	uEflags = pMixedCtx->eflags;
8811	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8812	uEflags.Bits.u1RF = 0;
8813	uEflags.Bits.u1VM = 0;
8814
8815	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8816	if (RT_FAILURE(rc))
8817	{
8818	rc = VERR_EM_INTERPRETER;
8819	break;
8820	}
8821	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8822	pMixedCtx->esp -= cbParm;
8823	pMixedCtx->esp &= uMask;
8824	pMixedCtx->rip += pDis->cbInstr;
8825	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8826	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8827	break;
8828	}
8829
8830	case OP_IRET:
8831	{
8832	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8833	* instruction reference. */
8834	RTGCPTR GCPtrStack = 0;
8835	uint32_t uMask = 0xffff;
8836	uint16_t aIretFrame[3];
8837	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8838	{
8839	rc = VERR_EM_INTERPRETER;
8840	break;
8841	}
8842	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8843	&GCPtrStack);
8844	if (RT_SUCCESS(rc))
8845	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8846	if (RT_FAILURE(rc))
8847	{
8848	rc = VERR_EM_INTERPRETER;
8849	break;
8850	}
8851	pMixedCtx->eip = 0;
8852	pMixedCtx->ip = aIretFrame[0];
8853	pMixedCtx->cs.Sel = aIretFrame[1];
8854	pMixedCtx->cs.ValidSel = aIretFrame[1];
8855	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8856	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8857	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8858	pMixedCtx->sp += sizeof(aIretFrame);
8859	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8860	\| HM_CHANGED_GUEST_RFLAGS;
8861	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8863	break;
8864	}
8865
8866	case OP_INT:
8867	{
8868	uint16_t uVector = pDis->Param1.uValue & 0xff;
8869	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8871	break;
8872	}
8873
8874	case OP_INTO:
8875	{
8876	if (pMixedCtx->eflags.Bits.u1OF)
8877	{
8878	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8879	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8880	}
8881	break;
8882	}
8883
8884	default:
8885	{
8886	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8887	EMCODETYPE_SUPERVISOR);
8888	rc = VBOXSTRICTRC_VAL(rc2);
8889	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8890	Log(("#GP rc=%Rrc\n", rc));
8891	break;
8892	}
8893	}
8894	}
8895	else
8896	rc = VERR_EM_INTERPRETER;
8897
8898	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8899	("#GP Unexpected rc=%Rrc\n", rc));
8900	return rc;
8901	}
8902
8903
8904	/**
8905	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8906	* the exception reported in the VMX transient structure back into the VM.
8907	*
8908	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8909	* up-to-date.
8910	*/
8911	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8912	{
8913	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8914
8915	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8916	hmR0VmxCheckExitDueToEventDelivery(). */
8917	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8918	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8919	AssertRCReturn(rc, rc);
8920	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8921	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8922	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8923	return VINF_SUCCESS;
8924	}
8925
8926
8927	/**
8928	* VM-exit exception handler for #PF (Page-fault exception).
8929	*/
8930	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8931	{
8932	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8933	PVM pVM = pVCpu->CTX_SUFF(pVM);
8934	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8935	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8936	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8937	AssertRCReturn(rc, rc);
8938
8939	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8940	if (pVM->hm.s.fNestedPaging)
8941	{
8942	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8943	{
8944	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8945	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8946	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8947	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8949	}
8950	else
8951	{
8952	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8953	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8954	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8955	Log(("Pending #DF due to vectoring #PF. NP\n"));
8956	}
8957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8958	return rc;
8959	}
8960	#else
8961	Assert(!pVM->hm.s.fNestedPaging);
8962	#endif
8963
8964	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8965	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8966	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8967	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8968	AssertRCReturn(rc, rc);
8969	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8970	if ( pVM->hm.s.fTRPPatchingAllowed
8971	&& pVM->hm.s.pGuestPatchMem
8972	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8973	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8974	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8975	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8976	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8977	{
8978	RTGCPHYS GCPhys;
8979	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8980	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8981	if ( rc == VINF_SUCCESS
8982	&& GCPhys == GCPhysApicBase)
8983	{
8984	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8985	AssertRCReturn(rc, rc);
8986
8987	/* Only attempt to patch the instruction once. */
8988	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8989	if (!pPatch)
8990	return VINF_EM_HM_PATCH_TPR_INSTR;
8991	}
8992	}
8993	#endif
8994
8995	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8996	AssertRCReturn(rc, rc);
8997
8998	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8999	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9000
9001	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9002	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9003	(RTGCPTR)pVmxTransient->uExitQualification);
9004
9005	Log(("#PF: rc=%Rrc\n", rc));
9006	if (rc == VINF_SUCCESS)
9007	{
9008	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9009	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9010	* memory? We don't update the whole state here... */
9011	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9012	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9013	TRPMResetTrap(pVCpu);
9014	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9015	return rc;
9016	}
9017	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9018	{
9019	if (!pVmxTransient->fVectoringPF)
9020	{
9021	/* It's a guest page fault and needs to be reflected to the guest. */
9022	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9023	TRPMResetTrap(pVCpu);
9024	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9025	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9026	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9027	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9028	}
9029	else
9030	{
9031	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9032	TRPMResetTrap(pVCpu);
9033	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9034	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9035	Log(("#PF: Pending #DF due to vectoring #PF\n"));
9036	}
9037
9038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9039	return VINF_SUCCESS;
9040	}
9041
9042	TRPMResetTrap(pVCpu);
9043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9044	return rc;
9045	}
9046

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45965

Download in other formats: