HMVMXR0.cpp@ 45892

Last change on this file since 45892 was 45892, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Some clarification regarding the unusable bit.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 376.3 KB

Line
1	/* $Id: HMVMXR0.cpp 45892 2013-05-03 11:44:10Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147	/** Profiling macro. */
148	#ifdef HM_PROFILE_EXIT_DISPATCH
149	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
150	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
151	#else
152	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
153	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
154	#endif
155
156
157	/*******************************************************************************
158	* Structures and Typedefs *
159	*******************************************************************************/
160	/**
161	* A state structure for holding miscellaneous information across
162	* VMX non-root operation and restored after the transition.
163	*/
164	typedef struct VMXTRANSIENT
165	{
166	/** The host's rflags/eflags. */
167	RTCCUINTREG uEFlags;
168	#if HC_ARCH_BITS == 32
169	uint32_t u32Alignment0;
170	#endif
171	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
172	uint64_t u64LStarMsr;
173	/** The guest's TPR value used for TPR shadowing. */
174	uint8_t u8GuestTpr;
175	/** Alignment. */
176	uint8_t abAlignment0[6];
177
178	/** The basic VM-exit reason. */
179	uint16_t uExitReason;
180	/** Alignment. */
181	uint16_t u16Alignment0;
182	/** The VM-exit interruption error code. */
183	uint32_t uExitIntrErrorCode;
184	/** The VM-exit exit qualification. */
185	RTGCUINTPTR uExitQualification;
186	#if GC_ARCH_BITS == 32
187	/** Alignment. */
188	uint32_t u32Alignment1;
189	#endif
190
191	/** The VM-exit interruption-information field. */
192	uint32_t uExitIntrInfo;
193	/** The VM-exit instruction-length field. */
194	uint32_t cbInstr;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Alignment. */
198	uint8_t abAlignment1[5];
199
200	/** The VM-entry interruption-information field. */
201	uint32_t uEntryIntrInfo;
202	/** The VM-entry exception error code field. */
203	uint32_t uEntryXcptErrorCode;
204	/** The VM-entry instruction length field. */
205	uint32_t cbEntryInstr;
206
207	/** IDT-vectoring information field. */
208	uint32_t uIdtVectoringInfo;
209	/** IDT-vectoring error code. */
210	uint32_t uIdtVectoringErrorCode;
211
212	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
213	uint32_t fVmcsFieldsRead;
214	/** Whether TSC-offsetting should be setup before VM-entry. */
215	bool fUpdateTscOffsettingAndPreemptTimer;
216	/** Whether the VM-exit was caused by a page-fault during delivery of a
217	* contributary exception or a page-fault. */
218	bool fVectoringPF;
219	} VMXTRANSIENT, *PVMXTRANSIENT;
220	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
221	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
222	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
223
224
225	/**
226	* MSR-bitmap read permissions.
227	*/
228	typedef enum VMXMSREXITREAD
229	{
230	/** Reading this MSR causes a VM-exit. */
231	VMXMSREXIT_INTERCEPT_READ = 0xb,
232	/** Reading this MSR does not cause a VM-exit. */
233	VMXMSREXIT_PASSTHRU_READ
234	} VMXMSREXITREAD;
235
236	/**
237	* MSR-bitmap write permissions.
238	*/
239	typedef enum VMXMSREXITWRITE
240	{
241	/** Writing to this MSR causes a VM-exit. */
242	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
243	/** Writing to this MSR does not cause a VM-exit. */
244	VMXMSREXIT_PASSTHRU_WRITE
245	} VMXMSREXITWRITE;
246
247
248	/*******************************************************************************
249	* Internal Functions *
250	*******************************************************************************/
251	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
252	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
253	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
254	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
255	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
256	#endif
257	#ifndef HMVMX_USE_FUNCTION_TABLE
258	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
259	#define HMVMX_EXIT_DECL static int
260	#else
261	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
262	#endif
263
264	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308
309	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316
317
318	/*******************************************************************************
319	* Global Variables *
320	*******************************************************************************/
321	#ifdef HMVMX_USE_FUNCTION_TABLE
322	/**
323	* VM-exit handler.
324	*
325	* @returns VBox status code.
326	* @param pVCpu Pointer to the VMCPU.
327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
328	* out-of-sync. Make sure to update the required
329	* fields before using them.
330	* @param pVmxTransient Pointer to the VMX-transient structure.
331	*/
332	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333	/** Pointer to VM-exit handler. */
334	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
335
336	/**
337	* VMX_EXIT dispatch table.
338	*/
339	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
340	{
341	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
342	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
343	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
344	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
345	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
346	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
347	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
348	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
349	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
350	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
351	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
352	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
353	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
354	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
355	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
356	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
357	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
358	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
359	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
360	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
361	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
362	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
363	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
364	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
365	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
366	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
367	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
368	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
369	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
370	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
371	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
372	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
373	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
374	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
375	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
376	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
377	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
378	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
379	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
380	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
381	/* 40 UNDEFINED */ hmR0VmxExitPause,
382	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
383	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
384	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
385	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
386	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
387	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
388	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
389	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
390	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
391	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
392	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
393	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
394	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
395	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
396	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
397	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
398	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
399	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
400	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
401	};
402	#endif /* HMVMX_USE_FUNCTION_TABLE */
403
404	#ifdef VBOX_STRICT
405	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
406	{
407	/* 0 */ "(Not Used)",
408	/* 1 */ "VMCALL executed in VMX root operation.",
409	/* 2 */ "VMCLEAR with invalid physical address.",
410	/* 3 */ "VMCLEAR with VMXON pointer.",
411	/* 4 */ "VMLAUNCH with non-clear VMCS.",
412	/* 5 */ "VMRESUME with non-launched VMCS.",
413	/* 6 */ "VMRESUME after VMXOFF",
414	/* 7 */ "VM entry with invalid control fields.",
415	/* 8 */ "VM entry with invalid host state fields.",
416	/* 9 */ "VMPTRLD with invalid physical address.",
417	/* 10 */ "VMPTRLD with VMXON pointer.",
418	/* 11 */ "VMPTRLD with incorrect revision identifier.",
419	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
420	/* 13 */ "VMWRITE to read-only VMCS component.",
421	/* 14 */ "(Not Used)",
422	/* 15 */ "VMXON executed in VMX root operation.",
423	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
424	/* 17 */ "VM entry with non-launched executing VMCS.",
425	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
426	/* 19 */ "VMCALL with non-clear VMCS.",
427	/* 20 */ "VMCALL with invalid VM-exit control fields.",
428	/* 21 */ "(Not Used)",
429	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
430	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
431	/* 24 */ "VMCALL with invalid SMM-monitor features.",
432	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
433	/* 26 */ "VM entry with events blocked by MOV SS.",
434	/* 27 */ "(Not Used)",
435	/* 28 */ "Invalid operand to INVEPT/INVVPID."
436	};
437	#endif /* VBOX_STRICT */
438
439
440
441	/**
442	* Updates the VM's last error record. If there was a VMX instruction error,
443	* reads the error data from the VMCS and updates VCPU's last error record as
444	* well.
445	*
446	* @param pVM Pointer to the VM.
447	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
448	* VERR_VMX_UNABLE_TO_START_VM or
449	* VERR_VMX_INVALID_VMCS_FIELD).
450	* @param rc The error code.
451	*/
452	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
453	{
454	AssertPtr(pVM);
455	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
456	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
457	{
458	AssertPtrReturnVoid(pVCpu);
459	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
460	}
461	pVM->hm.s.lLastError = rc;
462	}
463
464
465	/**
466	* Reads the VM-entry interruption-information field from the VMCS into the VMX
467	* transient structure.
468	*
469	* @returns VBox status code.
470	* @param pVmxTransient Pointer to the VMX transient structure.
471	*
472	* @remarks No-long-jump zone!!!
473	*/
474	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
475	{
476	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
477	AssertRCReturn(rc, rc);
478	return VINF_SUCCESS;
479	}
480
481
482	/**
483	* Reads the VM-entry exception error code field from the VMCS into
484	* the VMX transient structure.
485	*
486	* @returns VBox status code.
487	* @param pVmxTransient Pointer to the VMX transient structure.
488	*
489	* @remarks No-long-jump zone!!!
490	*/
491	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
492	{
493	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
494	AssertRCReturn(rc, rc);
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* Reads the VM-entry exception error code field from the VMCS into
501	* the VMX transient structure.
502	*
503	* @returns VBox status code.
504	* @param pVCpu Pointer to the VMCPU.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-exit interruption-information field from the VMCS into the VMX
519	* transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
528	{
529	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the VM-exit interruption error code from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVCpu Pointer to the VMCPU.
543	* @param pVmxTransient Pointer to the VMX transient structure.
544	*/
545	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
546	{
547	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
548	{
549	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
550	AssertRCReturn(rc, rc);
551	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
552	}
553	return VINF_SUCCESS;
554	}
555
556
557	/**
558	* Reads the VM-exit instruction length field from the VMCS into the VMX
559	* transient structure.
560	*
561	* @returns VBox status code.
562	* @param pVCpu Pointer to the VMCPU.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Reads the exit qualification from the VMCS into the VMX transient structure.
579	*
580	* @returns VBox status code.
581	* @param pVCpu Pointer to the VMCPU.
582	* @param pVmxTransient Pointer to the VMX transient structure.
583	*/
584	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
585	{
586	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
587	{
588	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
589	AssertRCReturn(rc, rc);
590	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
591	}
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the IDT-vectoring information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*
603	* @remarks No-long-jump zone!!!
604	*/
605	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
606	{
607	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
608	{
609	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
610	AssertRCReturn(rc, rc);
611	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
612	}
613	return VINF_SUCCESS;
614	}
615
616
617	/**
618	* Reads the IDT-vectoring error code from the VMCS into the VMX
619	* transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*/
624	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
625	{
626	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
629	AssertRCReturn(rc, rc);
630	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
631	}
632	return VINF_SUCCESS;
633	}
634
635
636	/**
637	* Enters VMX root mode operation on the current CPU.
638	*
639	* @returns VBox status code.
640	* @param pVM Pointer to the VM (optional, can be NULL, after
641	* a resume).
642	* @param HCPhysCpuPage Physical address of the VMXON region.
643	* @param pvCpuPage Pointer to the VMXON region.
644	*/
645	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
646	{
647	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
648	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
649	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
650
651	if (pVM)
652	{
653	/* Write the VMCS revision dword to the VMXON region. */
654	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
655	}
656
657	/* Enable the VMX bit in CR4 if necessary. */
658	RTCCUINTREG uCr4 = ASMGetCR4();
659	if (!(uCr4 & X86_CR4_VMXE))
660	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
661
662	/* Enter VMX root mode. */
663	int rc = VMXEnable(HCPhysCpuPage);
664	if (RT_FAILURE(rc))
665	ASMSetCR4(uCr4);
666
667	return rc;
668	}
669
670
671	/**
672	* Exits VMX root mode operation on the current CPU.
673	*
674	* @returns VBox status code.
675	*/
676	static int hmR0VmxLeaveRootMode(void)
677	{
678	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
679
680	/* If we're for some reason not in VMX root mode, then don't leave it. */
681	if (ASMGetCR4() & X86_CR4_VMXE)
682	{
683	/* Exit VMX root mode and clear the VMX bit in CR4 */
684	VMXDisable();
685	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
686	return VINF_SUCCESS;
687	}
688
689	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
690	}
691
692
693	/**
694	* Allocates and maps one physically contiguous page. The allocated page is
695	* zero'd out. (Used by various VT-x structures).
696	*
697	* @returns IPRT status code.
698	* @param pMemObj Pointer to the ring-0 memory object.
699	* @param ppVirt Where to store the virtual address of the
700	* allocation.
701	* @param pPhys Where to store the physical address of the
702	* allocation.
703	*/
704	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
705	{
706	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
707	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
708	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
709
710	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
711	if (RT_FAILURE(rc))
712	return rc;
713	ppVirt = RTR0MemObjAddress(pMemObj);
714	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
715	ASMMemZero32(*ppVirt, PAGE_SIZE);
716	return VINF_SUCCESS;
717	}
718
719
720	/**
721	* Frees and unmaps an allocated physical page.
722	*
723	* @param pMemObj Pointer to the ring-0 memory object.
724	* @param ppVirt Where to re-initialize the virtual address of
725	* allocation as 0.
726	* @param pHCPhys Where to re-initialize the physical address of the
727	* allocation as 0.
728	*/
729	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
730	{
731	AssertPtr(pMemObj);
732	AssertPtr(ppVirt);
733	AssertPtr(pHCPhys);
734	if (*pMemObj != NIL_RTR0MEMOBJ)
735	{
736	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
737	AssertRC(rc);
738	*pMemObj = NIL_RTR0MEMOBJ;
739	*ppVirt = 0;
740	*pHCPhys = 0;
741	}
742	}
743
744
745	/**
746	* Worker function to free VT-x related structures.
747	*
748	* @returns IPRT status code.
749	* @param pVM Pointer to the VM.
750	*/
751	static void hmR0VmxStructsFree(PVM pVM)
752	{
753	for (VMCPUID i = 0; i < pVM->cCpus; i++)
754	{
755	PVMCPU pVCpu = &pVM->aCpus[i];
756	AssertPtr(pVCpu);
757
758	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
759	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
761	#endif
762
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
765
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
768	}
769
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
772	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
773	#endif
774	}
775
776
777	/**
778	* Worker function to allocate VT-x related VM structures.
779	*
780	* @returns IPRT status code.
781	* @param pVM Pointer to the VM.
782	*/
783	static int hmR0VmxStructsAlloc(PVM pVM)
784	{
785	/*
786	* Initialize members up-front so we can cleanup properly on allocation failure.
787	*/
788	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
789	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
790	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
791	pVM->hm.s.vmx.HCPhys##a_Name = 0;
792
793	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
794	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
795	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
796	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
797
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
800	#endif
801	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
802
803	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
804	for (VMCPUID i = 0; i < pVM->cCpus; i++)
805	{
806	PVMCPU pVCpu = &pVM->aCpus[i];
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
813	#endif
814	}
815	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
816	#undef VMXLOCAL_INIT_VM_MEMOBJ
817
818	/*
819	* Allocate all the VT-x structures.
820	*/
821	int rc = VINF_SUCCESS;
822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
823	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
827	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
828	#endif
829
830	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
831	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
832	{
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
834	&pVM->hm.s.vmx.HCPhysApicAccess);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	/*
840	* Initialize per-VCPU VT-x structures.
841	*/
842	for (VMCPUID i =0; i < pVM->cCpus; i++)
843	{
844	PVMCPU pVCpu = &pVM->aCpus[i];
845	AssertPtr(pVCpu);
846
847	/* Allocate the VM control structure (VMCS). */
848	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
854	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
855	{
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
857	&pVCpu->hm.s.vmx.HCPhysVirtApic);
858	if (RT_FAILURE(rc))
859	goto cleanup;
860	}
861
862	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
864	{
865	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
866	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867	if (RT_FAILURE(rc))
868	goto cleanup;
869	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
870	}
871
872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
873	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
874	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875	if (RT_FAILURE(rc))
876	goto cleanup;
877
878	/* Allocate the VM-exit MSR-load page for the host MSRs. */
879	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	#endif
883	}
884
885	return VINF_SUCCESS;
886
887	cleanup:
888	hmR0VmxStructsFree(pVM);
889	return rc;
890	}
891
892
893	/**
894	* Does global VT-x initialization (called during module initialization).
895	*
896	* @returns VBox status code.
897	*/
898	VMMR0DECL(int) VMXR0GlobalInit(void)
899	{
900	#ifdef HMVMX_USE_FUNCTION_TABLE
901	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
902	# ifdef VBOX_STRICT
903	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
904	Assert(g_apfnVMExitHandlers[i]);
905	# endif
906	#endif
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Does global VT-x termination (called during module termination).
913	*/
914	VMMR0DECL(void) VMXR0GlobalTerm()
915	{
916	/* Nothing to do currently. */
917	}
918
919
920	/**
921	* Sets up and activates VT-x on the current CPU.
922	*
923	* @returns VBox status code.
924	* @param pCpu Pointer to the global CPU info struct.
925	* @param pVM Pointer to the VM (can be NULL after a host resume
926	* operation).
927	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
928	* fEnabledByHost is true).
929	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
930	* @a fEnabledByHost is true).
931	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
932	* enable VT-x/AMD-V on the host.
933	*/
934	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
935	{
936	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
937	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
938
939	if (!fEnabledByHost)
940	{
941	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
942	if (RT_FAILURE(rc))
943	return rc;
944	}
945
946	/*
947	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
948	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
949	* each time while reusing a VPID after hitting the MaxASID limit once.
950	*/
951	if ( pVM
952	&& pVM->hm.s.vmx.fVpid
953	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
954	{
955	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
956	pCpu->fFlushAsidBeforeUse = false;
957	}
958	else
959	pCpu->fFlushAsidBeforeUse = true;
960
961	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
962	++pCpu->cTlbFlushes;
963
964	return VINF_SUCCESS;
965	}
966
967
968	/**
969	* Deactivates VT-x on the current CPU.
970	*
971	* @returns VBox status code.
972	* @param pCpu Pointer to the global CPU info struct.
973	* @param pvCpuPage Pointer to the VMXON region.
974	* @param HCPhysCpuPage Physical address of the VMXON region.
975	*/
976	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
977	{
978	NOREF(pCpu);
979	NOREF(pvCpuPage);
980	NOREF(HCPhysCpuPage);
981
982	hmR0VmxLeaveRootMode();
983	return VINF_SUCCESS;
984	}
985
986
987	/**
988	* Sets the permission bits for the specified MSR in the MSR bitmap.
989	*
990	* @param pVCpu Pointer to the VMCPU.
991	* @param uMSR The MSR value.
992	* @param enmRead Whether reading this MSR causes a VM-exit.
993	* @param enmWrite Whether writing this MSR causes a VM-exit.
994	*/
995	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
996	{
997	int32_t iBit;
998	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
999
1000	/*
1001	* Layout:
1002	* 0x000 - 0x3ff - Low MSR read bits
1003	* 0x400 - 0x7ff - High MSR read bits
1004	* 0x800 - 0xbff - Low MSR write bits
1005	* 0xc00 - 0xfff - High MSR write bits
1006	*/
1007	if (uMsr <= 0x00001FFF)
1008	iBit = uMsr;
1009	else if ( uMsr >= 0xC0000000
1010	&& uMsr <= 0xC0001FFF)
1011	{
1012	iBit = (uMsr - 0xC0000000);
1013	pbMsrBitmap += 0x400;
1014	}
1015	else
1016	{
1017	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1018	return;
1019	}
1020
1021	Assert(iBit <= 0x1fff);
1022	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1023	ASMBitSet(pbMsrBitmap, iBit);
1024	else
1025	ASMBitClear(pbMsrBitmap, iBit);
1026
1027	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1028	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1029	else
1030	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1031	}
1032
1033
1034	/**
1035	* Flushes the TLB using EPT.
1036	*
1037	* @returns VBox status code.
1038	* @param pVM Pointer to the VM.
1039	* @param pVCpu Pointer to the VMCPU.
1040	* @param enmFlush Type of flush.
1041	*/
1042	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1043	{
1044	AssertPtr(pVM);
1045	Assert(pVM->hm.s.fNestedPaging);
1046
1047	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1048
1049	uint64_t descriptor[2];
1050	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1051	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1052
1053	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1054	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1055	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1056	}
1057
1058
1059	/**
1060	* Flushes the TLB using VPID.
1061	*
1062	* @returns VBox status code.
1063	* @param pVM Pointer to the VM.
1064	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1065	* enmFlush).
1066	* @param enmFlush Type of flush.
1067	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1068	* on @a enmFlush).
1069	*/
1070	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1071	{
1072	AssertPtr(pVM);
1073	Assert(pVM->hm.s.vmx.fVpid);
1074
1075	uint64_t descriptor[2];
1076	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1077	{
1078	descriptor[0] = 0;
1079	descriptor[1] = 0;
1080	}
1081	else
1082	{
1083	AssertPtr(pVCpu);
1084	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1085	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1086	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1087	descriptor[1] = GCPtr;
1088	}
1089
1090	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1091	AssertMsg(rc == VINF_SUCCESS,
1092	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1093	if ( RT_SUCCESS(rc)
1094	&& pVCpu)
1095	{
1096	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1097	}
1098	}
1099
1100
1101	/**
1102	* Invalidates a guest page by guest virtual address. Only relevant for
1103	* EPT/VPID, otherwise there is nothing really to invalidate.
1104	*
1105	* @returns VBox status code.
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU.
1108	* @param GCVirt Guest virtual address of the page to invalidate.
1109	*/
1110	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1111	{
1112	AssertPtr(pVM);
1113	AssertPtr(pVCpu);
1114	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1115
1116	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1117	if (!fFlushPending)
1118	{
1119	/*
1120	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1121	* See @bugref{6043} and @bugref{6177}.
1122	*
1123	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1124	* function maybe called in a loop with individual addresses.
1125	*/
1126	if (pVM->hm.s.vmx.fVpid)
1127	{
1128	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1129	{
1130	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1131	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1132	}
1133	else
1134	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1135	}
1136	else if (pVM->hm.s.fNestedPaging)
1137	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1138	}
1139
1140	return VINF_SUCCESS;
1141	}
1142
1143
1144	/**
1145	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1146	* otherwise there is nothing really to invalidate.
1147	*
1148	* @returns VBox status code.
1149	* @param pVM Pointer to the VM.
1150	* @param pVCpu Pointer to the VMCPU.
1151	* @param GCPhys Guest physical address of the page to invalidate.
1152	*/
1153	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1154	{
1155	LogFlowFunc(("%RGp\n", GCPhys));
1156
1157	/*
1158	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1159	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1160	* This function might be called in a loop.
1161	*/
1162	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1163	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1164	return VINF_SUCCESS;
1165	}
1166
1167
1168	/**
1169	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1170	* case where neither EPT nor VPID is supported by the CPU.
1171	*
1172	* @param pVM Pointer to the VM.
1173	* @param pVCpu Pointer to the VMCPU.
1174	*
1175	* @remarks Called with interrupts disabled.
1176	*/
1177	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1178	{
1179	NOREF(pVM);
1180	AssertPtr(pVCpu);
1181	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1182	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1183
1184	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1185	AssertPtr(pCpu);
1186
1187	pVCpu->hm.s.TlbShootdown.cPages = 0;
1188	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1189	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1190	pVCpu->hm.s.fForceTLBFlush = false;
1191	return;
1192	}
1193
1194
1195	/**
1196	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1197	*
1198	* @param pVM Pointer to the VM.
1199	* @param pVCpu Pointer to the VMCPU.
1200	* @remarks All references to "ASID" in this function pertains to "VPID" in
1201	* Intel's nomenclature. The reason is, to avoid confusion in compare
1202	* statements since the host-CPU copies are named "ASID".
1203	*
1204	* @remarks Called with interrupts disabled.
1205	*/
1206	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1207	{
1208	AssertPtr(pVM);
1209	AssertPtr(pVCpu);
1210	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1211	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1212	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1213
1214	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1215	AssertPtr(pCpu);
1216
1217	/*
1218	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1219	* This can happen both for start & resume due to long jumps back to ring-3.
1220	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1221	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1222	*/
1223	bool fNewASID = false;
1224	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1225	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1226	{
1227	pVCpu->hm.s.fForceTLBFlush = true;
1228	fNewASID = true;
1229	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1230	}
1231
1232	/*
1233	* Check for explicit TLB shootdowns.
1234	*/
1235	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1236	{
1237	pVCpu->hm.s.fForceTLBFlush = true;
1238	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1239	}
1240
1241	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1242	if (pVCpu->hm.s.fForceTLBFlush)
1243	{
1244	if (fNewASID)
1245	{
1246	++pCpu->uCurrentAsid;
1247	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1248	{
1249	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1250	pCpu->cTlbFlushes++;
1251	pCpu->fFlushAsidBeforeUse = true;
1252	}
1253
1254	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1255	if (pCpu->fFlushAsidBeforeUse)
1256	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1257	}
1258	else
1259	{
1260	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1261	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1262	else
1263	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1264	}
1265
1266	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1267	pVCpu->hm.s.fForceTLBFlush = false;
1268	}
1269	else
1270	{
1271	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1272	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1273	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1274	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1275
1276	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1277	* not be executed. See hmQueueInvlPage() where it is commented
1278	* out. Support individual entry flushing someday. */
1279	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1280	{
1281	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1282
1283	/*
1284	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1285	* as supported by the CPU.
1286	*/
1287	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1288	{
1289	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1290	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1291	}
1292	else
1293	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1294	}
1295	else
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1297	}
1298	pVCpu->hm.s.TlbShootdown.cPages = 0;
1299	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1300
1301	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1302	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1303	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1304	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1305	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1306	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1307
1308	/* Update VMCS with the VPID. */
1309	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1310	AssertRC(rc);
1311	}
1312
1313
1314	/**
1315	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1316	*
1317	* @returns VBox status code.
1318	* @param pVM Pointer to the VM.
1319	* @param pVCpu Pointer to the VMCPU.
1320	*
1321	* @remarks Called with interrupts disabled.
1322	*/
1323	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1324	{
1325	AssertPtr(pVM);
1326	AssertPtr(pVCpu);
1327	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1328	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1329
1330	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1331	AssertPtr(pCpu);
1332
1333	/*
1334	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1335	* This can happen both for start & resume due to long jumps back to ring-3.
1336	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1337	*/
1338	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1339	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1340	{
1341	pVCpu->hm.s.fForceTLBFlush = true;
1342	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1343	}
1344
1345	/* Check for explicit TLB shootdown flushes. */
1346	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1347	{
1348	pVCpu->hm.s.fForceTLBFlush = true;
1349	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1350	}
1351
1352	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1353	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1354
1355	if (pVCpu->hm.s.fForceTLBFlush)
1356	{
1357	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1358	pVCpu->hm.s.fForceTLBFlush = false;
1359	}
1360	else
1361	{
1362	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1363	* not be executed. See hmQueueInvlPage() where it is commented
1364	* out. Support individual entry flushing someday. */
1365	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1366	{
1367	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1369	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1370	}
1371	else
1372	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1373	}
1374
1375	pVCpu->hm.s.TlbShootdown.cPages = 0;
1376	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1377	}
1378
1379
1380	/**
1381	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1382	*
1383	* @returns VBox status code.
1384	* @param pVM Pointer to the VM.
1385	* @param pVCpu Pointer to the VMCPU.
1386	*
1387	* @remarks Called with interrupts disabled.
1388	*/
1389	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1390	{
1391	AssertPtr(pVM);
1392	AssertPtr(pVCpu);
1393	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1394	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1395
1396	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1397
1398	/*
1399	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1400	* This can happen both for start & resume due to long jumps back to ring-3.
1401	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1402	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1403	*/
1404	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1405	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1406	{
1407	pVCpu->hm.s.fForceTLBFlush = true;
1408	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1409	}
1410
1411	/* Check for explicit TLB shootdown flushes. */
1412	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1413	{
1414	/*
1415	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1416	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1417	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1418	*/
1419	pVCpu->hm.s.fForceTLBFlush = true;
1420	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1421	}
1422
1423	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1424	if (pVCpu->hm.s.fForceTLBFlush)
1425	{
1426	++pCpu->uCurrentAsid;
1427	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1428	{
1429	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1430	pCpu->fFlushAsidBeforeUse = true;
1431	pCpu->cTlbFlushes++;
1432	}
1433
1434	pVCpu->hm.s.fForceTLBFlush = false;
1435	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1436	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1437	if (pCpu->fFlushAsidBeforeUse)
1438	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1439	}
1440	else
1441	{
1442	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1443	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1444	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1445	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1446
1447	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1448	* not be executed. See hmQueueInvlPage() where it is commented
1449	* out. Support individual entry flushing someday. */
1450	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1451	{
1452	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1453	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1454	{
1455	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1456	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1457	}
1458	else
1459	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1460	}
1461	else
1462	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1463	}
1464
1465	pVCpu->hm.s.TlbShootdown.cPages = 0;
1466	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1467
1468	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1469	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1470	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1471	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1472	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1473	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1474
1475	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1476	AssertRC(rc);
1477	}
1478
1479
1480	/**
1481	* Flushes the guest TLB entry based on CPU capabilities.
1482	*
1483	* @param pVCpu Pointer to the VMCPU.
1484	*/
1485	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1486	{
1487	PVM pVM = pVCpu->CTX_SUFF(pVM);
1488	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1489	{
1490	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1491	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1492	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1493	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1494	default:
1495	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1496	break;
1497	}
1498	}
1499
1500
1501	/**
1502	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1503	* TLB entries from the host TLB before VM-entry.
1504	*
1505	* @returns VBox status code.
1506	* @param pVM Pointer to the VM.
1507	*/
1508	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1509	{
1510	/*
1511	* Determine optimal flush type for nested paging.
1512	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1513	* guest execution (see hmR3InitFinalizeR0()).
1514	*/
1515	if (pVM->hm.s.fNestedPaging)
1516	{
1517	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1518	{
1519	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1520	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1521	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1522	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1523	else
1524	{
1525	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1526	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1527	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1528	}
1529
1530	/* Make sure the write-back cacheable memory type for EPT is supported. */
1531	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1532	{
1533	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1534	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1535	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1536	}
1537	}
1538	else
1539	{
1540	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1541	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1542	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1543	}
1544	}
1545
1546	/*
1547	* Determine optimal flush type for VPID.
1548	*/
1549	if (pVM->hm.s.vmx.fVpid)
1550	{
1551	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1552	{
1553	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1554	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1555	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1556	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1557	else
1558	{
1559	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1560	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1561	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1562	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1563	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1564	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1565	pVM->hm.s.vmx.fVpid = false;
1566	}
1567	}
1568	else
1569	{
1570	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1571	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1572	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1573	pVM->hm.s.vmx.fVpid = false;
1574	}
1575	}
1576
1577	/*
1578	* Setup the handler for flushing tagged-TLBs.
1579	*/
1580	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1581	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1582	else if (pVM->hm.s.fNestedPaging)
1583	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1584	else if (pVM->hm.s.vmx.fVpid)
1585	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1586	else
1587	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1588	return VINF_SUCCESS;
1589	}
1590
1591
1592	/**
1593	* Sets up pin-based VM-execution controls in the VMCS.
1594	*
1595	* @returns VBox status code.
1596	* @param pVM Pointer to the VM.
1597	* @param pVCpu Pointer to the VMCPU.
1598	*/
1599	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1600	{
1601	AssertPtr(pVM);
1602	AssertPtr(pVCpu);
1603
1604	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1605	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1606
1607	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1608	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1609	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1610
1611	/* Enable the VMX preemption timer. */
1612	if (pVM->hm.s.vmx.fUsePreemptTimer)
1613	{
1614	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1615	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1616	}
1617
1618	if ((val & zap) != val)
1619	{
1620	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1621	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1622	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1623	}
1624
1625	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1626	AssertRCReturn(rc, rc);
1627
1628	/* Update VCPU with the currently set pin-based VM-execution controls. */
1629	pVCpu->hm.s.vmx.u32PinCtls = val;
1630	return rc;
1631	}
1632
1633
1634	/**
1635	* Sets up processor-based VM-execution controls in the VMCS.
1636	*
1637	* @returns VBox status code.
1638	* @param pVM Pointer to the VM.
1639	* @param pVMCPU Pointer to the VMCPU.
1640	*/
1641	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1642	{
1643	AssertPtr(pVM);
1644	AssertPtr(pVCpu);
1645
1646	int rc = VERR_INTERNAL_ERROR_5;
1647	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1648	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1649
1650	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1654	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1655	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1656	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1657
1658	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1659	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1660	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1661	{
1662	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1663	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1664	}
1665
1666	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1667	if (!pVM->hm.s.fNestedPaging)
1668	{
1669	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1670	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1671	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1672	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1673	}
1674
1675	/* Use TPR shadowing if supported by the CPU. */
1676	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1677	{
1678	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1679	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1680	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1681	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1682	AssertRCReturn(rc, rc);
1683
1684	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1685	/* CR8 writes causes a VM-exit based on TPR threshold. */
1686	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1687	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1688	}
1689	else
1690	{
1691	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1692	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1693	}
1694
1695	/* Use MSR-bitmaps if supported by the CPU. */
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1697	{
1698	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1699
1700	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1701	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1702	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1703	AssertRCReturn(rc, rc);
1704
1705	/*
1706	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1707	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1708	*/
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1716	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1717	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1718	}
1719
1720	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1722	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1723
1724	if ((val & zap) != val)
1725	{
1726	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1727	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1728	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1729	}
1730
1731	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1732	AssertRCReturn(rc, rc);
1733
1734	/* Update VCPU with the currently set processor-based VM-execution controls. */
1735	pVCpu->hm.s.vmx.u32ProcCtls = val;
1736
1737	/*
1738	* Secondary processor-based VM-execution controls.
1739	*/
1740	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1741	{
1742	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1743	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1744
1745	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1747
1748	if (pVM->hm.s.fNestedPaging)
1749	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1750	else
1751	{
1752	/*
1753	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1754	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1755	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1756	*/
1757	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1758	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1759	}
1760
1761	if (pVM->hm.s.vmx.fVpid)
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1763
1764	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1765	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1766
1767	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1768	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1769	* done dynamically. */
1770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1771	{
1772	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1773	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1774	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1775	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1776	AssertRCReturn(rc, rc);
1777	}
1778
1779	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1780	{
1781	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1782	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1783	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1784	}
1785
1786	if ((val & zap) != val)
1787	{
1788	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1789	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1790	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1791	}
1792
1793	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1794	AssertRCReturn(rc, rc);
1795
1796	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1797	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1798	}
1799
1800	return VINF_SUCCESS;
1801	}
1802
1803
1804	/**
1805	* Sets up miscellaneous (everything other than Pin & Processor-based
1806	* VM-execution) control fields in the VMCS.
1807	*
1808	* @returns VBox status code.
1809	* @param pVM Pointer to the VM.
1810	* @param pVCpu Pointer to the VMCPU.
1811	*/
1812	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1813	{
1814	AssertPtr(pVM);
1815	AssertPtr(pVCpu);
1816
1817	int rc = VERR_GENERAL_FAILURE;
1818
1819	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1820	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1821
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1823
1824	/*
1825	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1826	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1827	* We thus use the exception bitmap to control it rather than use both.
1828	*/
1829	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1830	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1831
1832	/** @todo Explore possibility of using IO-bitmaps. */
1833	/* All IO & IOIO instructions cause VM-exits. */
1834	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1835	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1836
1837	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1838	/* Setup MSR autoloading/autostoring. */
1839	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1840	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1841	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1843	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1844	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1845
1846	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1847	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1848	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1849	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1850	#else
1851	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1852	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1853	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1854	#endif
1855
1856	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1857	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1858
1859	/* Setup debug controls */
1860	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1861	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1862	AssertRCReturn(rc, rc);
1863	return rc;
1864	}
1865
1866
1867	/**
1868	* Sets up the initial exception bitmap in the VMCS based on static conditions
1869	* (i.e. conditions that cannot ever change at runtime).
1870	*
1871	* @returns VBox status code.
1872	* @param pVM Pointer to the VM.
1873	* @param pVCpu Pointer to the VMCPU.
1874	*/
1875	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1876	{
1877	AssertPtr(pVM);
1878	AssertPtr(pVCpu);
1879
1880	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1881
1882	uint32_t u32XcptBitmap = 0;
1883
1884	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1885	if (!pVM->hm.s.fNestedPaging)
1886	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1887
1888	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1890	AssertRCReturn(rc, rc);
1891	return rc;
1892	}
1893
1894
1895	/**
1896	* Sets up the initial guest-state mask. The guest-state mask is consulted
1897	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1898	* for the nested virtualization case (as it would cause a VM-exit).
1899	*
1900	* @param pVCpu Pointer to the VMCPU.
1901	*/
1902	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1903	{
1904	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1905	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Does per-VM VT-x initialization.
1912	*
1913	* @returns VBox status code.
1914	* @param pVM Pointer to the VM.
1915	*/
1916	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1917	{
1918	LogFlowFunc(("pVM=%p\n", pVM));
1919
1920	int rc = hmR0VmxStructsAlloc(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	/**
1932	* Does per-VM VT-x termination.
1933	*
1934	* @returns VBox status code.
1935	* @param pVM Pointer to the VM.
1936	*/
1937	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1938	{
1939	LogFlowFunc(("pVM=%p\n", pVM));
1940
1941	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1942	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1943	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1944	#endif
1945	hmR0VmxStructsFree(pVM);
1946	return VINF_SUCCESS;
1947	}
1948
1949
1950	/**
1951	* Sets up the VM for execution under VT-x.
1952	* This function is only called once per-VM during initalization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1958	{
1959	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1961
1962	LogFlowFunc(("pVM=%p\n", pVM));
1963
1964	/*
1965	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1966	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1967	*/
1968	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1969	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1970	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1971	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1972	{
1973	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1974	return VERR_INTERNAL_ERROR;
1975	}
1976
1977	/* Initialize these always, see hmR3InitFinalizeR0().*/
1978	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1979	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1980
1981	/* Setup the tagged-TLB flush handlers. */
1982	int rc = hmR0VmxSetupTaggedTlb(pVM);
1983	if (RT_FAILURE(rc))
1984	{
1985	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1986	return rc;
1987	}
1988
1989	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1990	{
1991	PVMCPU pVCpu = &pVM->aCpus[i];
1992	AssertPtr(pVCpu);
1993	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1994
1995	/* Set revision dword at the beginning of the VMCS structure. */
1996	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1997
1998	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1999	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2000	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2001	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2002
2003	/* Load this VMCS as the current VMCS. */
2004	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2017	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2018	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2019
2020	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2021	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2022	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2023
2024	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2025	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2026	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2027
2028	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2029	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2030	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2031	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2032	#endif
2033
2034	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2035	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2036	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2037	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2038
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2040	}
2041
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2048	* the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	* @param pVCpu Pointer to the VMCPU.
2053	*/
2054	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2055	{
2056	RTCCUINTREG uReg = ASMGetCR0();
2057	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2058
2059	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2060	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2061	if (HMVMX_IS_64BIT_HOST_MODE())
2062	{
2063	uint64_t uRegCR3 = hmR0Get64bitCR3();
2064	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2065	}
2066	else
2067	#endif
2068	{
2069	uReg = ASMGetCR3();
2070	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2071	}
2072
2073	uReg = ASMGetCR4();
2074	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2075	AssertRCReturn(rc, rc);
2076	return rc;
2077	}
2078
2079
2080	/**
2081	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2082	* the host-state area in the VMCS.
2083	*
2084	* @returns VBox status code.
2085	* @param pVM Pointer to the VM.
2086	* @param pVCpu Pointer to the VMCPU.
2087	*/
2088	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2089	{
2090	int rc = VERR_INTERNAL_ERROR_5;
2091	RTSEL uSelCS = 0;
2092	RTSEL uSelSS = 0;
2093	RTSEL uSelDS = 0;
2094	RTSEL uSelES = 0;
2095	RTSEL uSelFS = 0;
2096	RTSEL uSelGS = 0;
2097	RTSEL uSelTR = 0;
2098
2099	/*
2100	* Host Selector registers.
2101	*/
2102	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2103	if (HMVMX_IS_64BIT_HOST_MODE())
2104	{
2105	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2106	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2107	}
2108	else
2109	{
2110	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2111	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2112	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2113	}
2114	#else
2115	uSelCS = ASMGetCS();
2116	uSelSS = ASMGetSS();
2117	#endif
2118
2119	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2120	uSelTR = ASMGetTR();
2121
2122	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2123	/** @todo Verify if we have any platform that actually run with DS or ES with
2124	* RPL != 0 in kernel space. */
2125	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2126	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2127	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2128	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2129	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2130	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2131	Assert(uSelCS != 0);
2132	Assert(uSelTR != 0);
2133
2134	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2135	#if 0
2136	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2137	Assert(uSelSS != 0);
2138	#endif
2139
2140	/* Write these host selector fields into the host-state area in the VMCS. */
2141	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2142	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2143	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2144	#if 0
2145	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2146	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2147	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2148	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2149	#endif
2150	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2151	AssertRCReturn(rc, rc);
2152
2153	/*
2154	* Host GDTR and IDTR.
2155	*/
2156	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2157	* be safe to -not- save and restore GDTR and IDTR in the assembly
2158	* code and just do it here and don't care if the limits are zapped on
2159	* VM-exit. */
2160	RTGDTR Gdtr;
2161	RT_ZERO(Gdtr);
2162	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2163	if (HMVMX_IS_64BIT_HOST_MODE())
2164	{
2165	X86XDTR64 Gdtr64;
2166	X86XDTR64 Idtr64;
2167	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2168	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2169	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2170	Gdtr.cbGdt = Gdtr64.cb;
2171	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2172	}
2173	else
2174	#endif
2175	{
2176	RTIDTR Idtr;
2177	ASMGetGDTR(&Gdtr);
2178	ASMGetIDTR(&Idtr);
2179	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2180	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2181	}
2182	AssertRCReturn(rc, rc);
2183
2184	/*
2185	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2186	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2187	*/
2188	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2189	{
2190	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2191	return VERR_VMX_INVALID_HOST_STATE;
2192	}
2193
2194	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2195	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2196	if (HMVMX_IS_64BIT_HOST_MODE())
2197	{
2198	/* We need the 64-bit TR base for hybrid darwin. */
2199	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2200	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2201	}
2202	else
2203	#endif
2204	{
2205	uintptr_t uTRBase;
2206	#if HC_ARCH_BITS == 64
2207	uTRBase = X86DESC64_BASE(pDesc);
2208	#else
2209	uTRBase = X86DESC_BASE(pDesc);
2210	#endif
2211	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2212	}
2213	AssertRCReturn(rc, rc);
2214
2215	/*
2216	* Host FS base and GS base.
2217	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2218	* would take care of the bases. In 64-bit, the MSRs come into play.
2219	*/
2220	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2221	if (HMVMX_IS_64BIT_HOST_MODE())
2222	{
2223	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2224	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2225	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2226	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2227	AssertRCReturn(rc, rc);
2228	}
2229	#endif
2230	return rc;
2231	}
2232
2233
2234	/**
2235	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2236	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2237	* the host after every successful VM exit.
2238	*
2239	* @returns VBox status code.
2240	* @param pVM Pointer to the VM.
2241	* @param pVCpu Pointer to the VMCPU.
2242	*/
2243	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2244	{
2245	AssertPtr(pVCpu);
2246	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2247
2248	int rc = VINF_SUCCESS;
2249	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2250	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2251	uint32_t cHostMsrs = 0;
2252	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2253
2254	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2255	{
2256	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2257	pHostMsr->u32Reserved = 0;
2258	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2259	if (CPUMIsGuestInLongMode(pVCpu))
2260	{
2261	/* Must match the EFER value in our 64 bits switcher. */
2262	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2263	}
2264	else
2265	# endif
2266	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2267	pHostMsr++; cHostMsrs++;
2268	}
2269
2270	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2271	if (HMVMX_IS_64BIT_HOST_MODE())
2272	{
2273	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2274	pHostMsr->u32Reserved = 0;
2275	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2276	pHostMsr++; cHostMsrs++;
2277	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2278	pHostMsr->u32Reserved = 0;
2279	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2280	pHostMsr++; cHostMsrs++;
2281	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2282	pHostMsr->u32Reserved = 0;
2283	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2284	pHostMsr++; cHostMsrs++;
2285	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2286	pHostMsr->u32Reserved = 0;
2287	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2288	pHostMsr++; cHostMsrs++;
2289	}
2290	# endif
2291
2292	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2293	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2294	{
2295	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2297	}
2298
2299	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2300	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2301
2302	/*
2303	* Host Sysenter MSRs.
2304	*/
2305	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2306	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2307	if (HMVMX_IS_64BIT_HOST_MODE())
2308	{
2309	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2310	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2311	}
2312	else
2313	{
2314	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2315	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2316	}
2317	# elif HC_ARCH_BITS == 32
2318	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2319	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2320	# else
2321	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2322	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2323	# endif
2324	AssertRCReturn(rc, rc);
2325
2326	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2327	* hmR0VmxSetupExitCtls() !! */
2328	return rc;
2329	}
2330
2331
2332	/**
2333	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2334	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2335	* controls".
2336	*
2337	* @returns VBox status code.
2338	* @param pVCpu Pointer to the VMCPU.
2339	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2340	* out-of-sync. Make sure to update the required fields
2341	* before using them.
2342	*
2343	* @remarks No-long-jump zone!!!
2344	*/
2345	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2346	{
2347	int rc = VINF_SUCCESS;
2348	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2349	{
2350	PVM pVM = pVCpu->CTX_SUFF(pVM);
2351	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2355	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2356
2357	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2358	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2359	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2360	else
2361	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2362
2363	/*
2364	* The following should not be set (since we're not in SMM mode):
2365	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2366	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2367	*/
2368
2369	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2370	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2371	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2372
2373	if ((val & zap) != val)
2374	{
2375	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2376	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2377	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2378	}
2379
2380	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2381	AssertRCReturn(rc, rc);
2382
2383	/* Update VCPU with the currently set VM-exit controls. */
2384	pVCpu->hm.s.vmx.u32EntryCtls = val;
2385	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2386	}
2387	return rc;
2388	}
2389
2390
2391	/**
2392	* Sets up the VM-exit controls in the VMCS.
2393	*
2394	* @returns VBox status code.
2395	* @param pVM Pointer to the VM.
2396	* @param pVCpu Pointer to the VMCPU.
2397	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2398	* out-of-sync. Make sure to update the required fields
2399	* before using them.
2400	*
2401	* @remarks requires EFER.
2402	*/
2403	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2404	{
2405	int rc = VINF_SUCCESS;
2406	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2407	{
2408	PVM pVM = pVCpu->CTX_SUFF(pVM);
2409	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2410	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2411
2412	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2413	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2414
2415	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2416	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2417	if (HMVMX_IS_64BIT_HOST_MODE())
2418	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2419	else
2420	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2421	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2422	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2423	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2424	else
2425	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2426	#endif
2427
2428	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2429	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2430
2431	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2432	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2433	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2434	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2435	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2436
2437	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2438	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2439
2440	if ((val & zap) != val)
2441	{
2442	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2443	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2444	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2445	}
2446
2447	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2448	AssertRCReturn(rc, rc);
2449
2450	/* Update VCPU with the currently set VM-exit controls. */
2451	pVCpu->hm.s.vmx.u32ExitCtls = val;
2452	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2453	}
2454	return rc;
2455	}
2456
2457
2458	/**
2459	* Loads the guest APIC and related state.
2460	*
2461	* @returns VBox status code.
2462	* @param pVM Pointer to the VM.
2463	* @param pVCpu Pointer to the VMCPU.
2464	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2465	* out-of-sync. Make sure to update the required fields
2466	* before using them.
2467	*/
2468	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2469	{
2470	int rc = VINF_SUCCESS;
2471	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2472	{
2473	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2474	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2475	{
2476	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2477
2478	bool fPendingIntr = false;
2479	uint8_t u8GuestTpr = 0;
2480	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2481	AssertRCReturn(rc, rc);
2482
2483	/*
2484	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2485	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2486	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2487	* the interrupt when we VM-exit for other reasons.
2488	*/
2489	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2490	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2491	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2492	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2493
2494	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2495	AssertRCReturn(rc, rc);
2496
2497	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2498	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2499	{
2500	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2501	pMixedCtx->msrLSTAR = u8GuestTpr;
2502	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2503	{
2504	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2505	if (fPendingIntr)
2506	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2507	else
2508	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2509	}
2510	}
2511	}
2512
2513	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2514	}
2515	return rc;
2516	}
2517
2518
2519	/**
2520	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2521	*
2522	* @returns
2523	* @param pVCpu Pointer to the VMCPU.
2524	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2525	* out-of-sync. Make sure to update the required fields
2526	* before using them.
2527	*
2528	* @remarks No-long-jump zone!!!
2529	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2530	*/
2531	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2532	{
2533	/*
2534	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2535	* inhibit interrupts or clear any existing interrupt-inhibition.
2536	*/
2537	uint32_t uIntrState = 0;
2538	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2539	{
2540	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2541	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2542	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2543	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2544	{
2545	/*
2546	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2547	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2548	*/
2549	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2550	}
2551	else if (pMixedCtx->eflags.Bits.u1IF)
2552	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2553	else
2554	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2555	}
2556	return uIntrState;
2557	}
2558
2559
2560	/**
2561	* Loads the guest's interruptibility-state into the guest-state area in the
2562	* VMCS.
2563	*
2564	* @returns VBox status code.
2565	* @param pVCpu Pointer to the VMCPU.
2566	* @param uIntrState The interruptibility-state to set.
2567	*/
2568	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2569	{
2570	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2571	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2572	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2573	AssertRCReturn(rc, rc);
2574	return rc;
2575	}
2576
2577
2578	/**
2579	* Loads the guest's RIP into the guest-state area in the VMCS.
2580	*
2581	* @returns VBox status code.
2582	* @param pVCpu Pointer to the VMCPU.
2583	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2584	* out-of-sync. Make sure to update the required fields
2585	* before using them.
2586	*
2587	* @remarks No-long-jump zone!!!
2588	*/
2589	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2590	{
2591	int rc = VINF_SUCCESS;
2592	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2593	{
2594	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2595	AssertRCReturn(rc, rc);
2596	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2597	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2598	}
2599	return rc;
2600	}
2601
2602
2603	/**
2604	* Loads the guest's RSP into the guest-state area in the VMCS.
2605	*
2606	* @returns VBox status code.
2607	* @param pVCpu Pointer to the VMCPU.
2608	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2609	* out-of-sync. Make sure to update the required fields
2610	* before using them.
2611	*
2612	* @remarks No-long-jump zone!!!
2613	*/
2614	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2615	{
2616	int rc = VINF_SUCCESS;
2617	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2618	{
2619	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2620	AssertRCReturn(rc, rc);
2621	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2622	}
2623	return rc;
2624	}
2625
2626
2627	/**
2628	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2629	*
2630	* @returns VBox status code.
2631	* @param pVCpu Pointer to the VMCPU.
2632	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2633	* out-of-sync. Make sure to update the required fields
2634	* before using them.
2635	*
2636	* @remarks No-long-jump zone!!!
2637	*/
2638	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2639	{
2640	int rc = VINF_SUCCESS;
2641	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2642	{
2643	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2644	Let us assert it as such and use 32-bit VMWRITE. */
2645	Assert(!(pMixedCtx->rflags.u64 >> 32));
2646	X86EFLAGS uEFlags = pMixedCtx->eflags;
2647	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2648	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2649
2650	/*
2651	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2652	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2653	*/
2654	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2655	{
2656	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2657	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2658	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2659	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2660	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2661	}
2662
2663	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2664	AssertRCReturn(rc, rc);
2665
2666	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2667	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2668	}
2669	return rc;
2670	}
2671
2672
2673	/**
2674	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2675	*
2676	* @returns VBox status code.
2677	* @param pVCpu Pointer to the VMCPU.
2678	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2679	* out-of-sync. Make sure to update the required fields
2680	* before using them.
2681	*
2682	* @remarks No-long-jump zone!!!
2683	*/
2684	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2685	{
2686	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2687	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2688	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2689	return rc;
2690	}
2691
2692
2693	/**
2694	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2695	* in the VMCS.
2696	*
2697	* @returns VBox status code.
2698	* @param pVM Pointer to the VM.
2699	* @param pVCpu Pointer to the VMCPU.
2700	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2701	* out-of-sync. Make sure to update the required fields
2702	* before using them.
2703	*
2704	* @remarks No-long-jump zone!!!
2705	*/
2706	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2707	{
2708	int rc = VINF_SUCCESS;
2709	PVM pVM = pVCpu->CTX_SUFF(pVM);
2710
2711	/*
2712	* Guest CR0.
2713	* Guest FPU.
2714	*/
2715	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2716	{
2717	Assert(!(pCtx->cr0 >> 32));
2718	uint32_t u32GuestCR0 = pCtx->cr0;
2719
2720	/* The guest's view (read access) of its CR0 is unblemished. */
2721	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2722	AssertRCReturn(rc, rc);
2723	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2724
2725	/* Setup VT-x's view of the guest CR0. */
2726	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2727	if (pVM->hm.s.fNestedPaging)
2728	{
2729	if (CPUMIsGuestPagingEnabledEx(pCtx))
2730	{
2731	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2732	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2733	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2734	}
2735	else
2736	{
2737	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2738	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2739	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2740	}
2741
2742	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2743	AssertRCReturn(rc, rc);
2744	}
2745	else
2746	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2747
2748	/*
2749	* Guest FPU bits.
2750	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2751	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2752	*/
2753	u32GuestCR0 \|= X86_CR0_NE;
2754	bool fInterceptNM = false;
2755	if (CPUMIsGuestFPUStateActive(pVCpu))
2756	{
2757	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2758	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2759	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2760	}
2761	else
2762	{
2763	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2764	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2765	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2766	}
2767
2768	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2769	bool fInterceptMF = false;
2770	if (!(pCtx->cr0 & X86_CR0_NE))
2771	fInterceptMF = true;
2772
2773	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2774	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2775	{
2776	Assert(PDMVmmDevHeapIsEnabled(pVM));
2777	Assert(pVM->hm.s.vmx.pRealModeTSS);
2778	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2779	fInterceptNM = true;
2780	fInterceptMF = true;
2781	}
2782	else
2783	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2784
2785	if (fInterceptNM)
2786	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2787	else
2788	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2789
2790	if (fInterceptMF)
2791	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2792	else
2793	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2794
2795	/* Additional intercepts for debugging, define these yourself explicitly. */
2796	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2797	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2798	\| RT_BIT(X86_XCPT_DB)
2799	\| RT_BIT(X86_XCPT_DE)
2800	\| RT_BIT(X86_XCPT_NM)
2801	\| RT_BIT(X86_XCPT_UD)
2802	\| RT_BIT(X86_XCPT_NP)
2803	\| RT_BIT(X86_XCPT_SS)
2804	\| RT_BIT(X86_XCPT_GP)
2805	\| RT_BIT(X86_XCPT_PF)
2806	\| RT_BIT(X86_XCPT_MF);
2807	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2808	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2809	#endif
2810
2811	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2812
2813	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2814	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2815	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2816	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2817	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2818	else
2819	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2820
2821	u32GuestCR0 \|= uSetCR0;
2822	u32GuestCR0 &= uZapCR0;
2823	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2824
2825	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2826	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2827	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2828	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2829
2830	/*
2831	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2832	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2833	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2834	*/
2835	uint64_t u64CR0Mask = 0;
2836	u64CR0Mask = X86_CR0_PE
2837	\| X86_CR0_NE
2838	\| X86_CR0_WP
2839	\| X86_CR0_PG
2840	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2841	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2842	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2843	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2844	u64CR0Mask &= ~X86_CR0_PE;
2845	if (pVM->hm.s.fNestedPaging)
2846	u64CR0Mask &= ~X86_CR0_WP;
2847
2848	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2849	if (fInterceptNM)
2850	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2851	else
2852	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2853
2854	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2855	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2856	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2857	AssertRCReturn(rc, rc);
2858
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2860	}
2861
2862	/*
2863	* Guest CR2.
2864	* It's always loaded in the assembler code. Nothing to do here.
2865	*/
2866
2867	/*
2868	* Guest CR3.
2869	*/
2870	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2871	{
2872	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2873	if (pVM->hm.s.fNestedPaging)
2874	{
2875	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2876
2877	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2878	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2879	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2880	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2881
2882	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2883	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2884	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2885
2886	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2887	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2888	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2889	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2890
2891	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2892	AssertRCReturn(rc, rc);
2893	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2894
2895	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2896	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2897	{
2898	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2899	if (CPUMIsGuestInPAEModeEx(pCtx))
2900	{
2901	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2902	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2903	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2904	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2905	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2906	AssertRCReturn(rc, rc);
2907	}
2908
2909	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2910	have Unrestricted Execution to handle the guest when it's not using paging. */
2911	GCPhysGuestCR3 = pCtx->cr3;
2912	}
2913	else
2914	{
2915	/*
2916	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2917	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2918	* EPT takes care of translating it to host-physical addresses.
2919	*/
2920	RTGCPHYS GCPhys;
2921	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2922	Assert(PDMVmmDevHeapIsEnabled(pVM));
2923
2924	/* We obtain it here every time as the guest could have relocated this PCI region. */
2925	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2926	AssertRCReturn(rc, rc);
2927
2928	GCPhysGuestCR3 = GCPhys;
2929	}
2930	}
2931	else
2932	{
2933	/* Non-nested paging case, just use the hypervisor's CR3. */
2934	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2935	}
2936
2937	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2938	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2939	AssertRCReturn(rc, rc);
2940
2941	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2942	}
2943
2944	/*
2945	* Guest CR4.
2946	*/
2947	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2948	{
2949	Assert(!(pCtx->cr4 >> 32));
2950	uint32_t u32GuestCR4 = pCtx->cr4;
2951
2952	/* The guest's view of its CR4 is unblemished. */
2953	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2954	AssertRCReturn(rc, rc);
2955	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2956
2957	/* Setup VT-x's view of the guest CR4. */
2958	/*
2959	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2960	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2961	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2962	*/
2963	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2964	{
2965	Assert(pVM->hm.s.vmx.pRealModeTSS);
2966	Assert(PDMVmmDevHeapIsEnabled(pVM));
2967	u32GuestCR4 &= ~X86_CR4_VME;
2968	}
2969
2970	if (pVM->hm.s.fNestedPaging)
2971	{
2972	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2973	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2974	{
2975	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2976	u32GuestCR4 \|= X86_CR4_PSE;
2977	/* Our identity mapping is a 32 bits page directory. */
2978	u32GuestCR4 &= ~X86_CR4_PAE;
2979	}
2980	/* else use guest CR4.*/
2981	}
2982	else
2983	{
2984	/*
2985	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2986	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2987	*/
2988	switch (pVCpu->hm.s.enmShadowMode)
2989	{
2990	case PGMMODE_REAL: /* Real-mode. */
2991	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2992	case PGMMODE_32_BIT: /* 32-bit paging. */
2993	{
2994	u32GuestCR4 &= ~X86_CR4_PAE;
2995	break;
2996	}
2997
2998	case PGMMODE_PAE: /* PAE paging. */
2999	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3000	{
3001	u32GuestCR4 \|= X86_CR4_PAE;
3002	break;
3003	}
3004
3005	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3006	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3007	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3008	break;
3009	#endif
3010	default:
3011	AssertFailed();
3012	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3013	}
3014	}
3015
3016	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3017	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3018	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3019	u32GuestCR4 \|= uSetCR4;
3020	u32GuestCR4 &= uZapCR4;
3021
3022	/* Write VT-x's view of the guest CR4 into the VMCS. */
3023	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3024	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3025
3026	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3027	uint64_t u64CR4Mask = 0;
3028	u64CR4Mask = X86_CR4_VME
3029	\| X86_CR4_PAE
3030	\| X86_CR4_PGE
3031	\| X86_CR4_PSE
3032	\| X86_CR4_VMXE;
3033	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3034	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3035	AssertRCReturn(rc, rc);
3036
3037	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3038	}
3039	return rc;
3040	}
3041
3042
3043	/**
3044	* Loads the guest debug registers into the guest-state area in the VMCS.
3045	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3046	*
3047	* @returns VBox status code.
3048	* @param pVCpu Pointer to the VMCPU.
3049	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3050	* out-of-sync. Make sure to update the required fields
3051	* before using them.
3052	*
3053	* @remarks No-long-jump zone!!!
3054	*/
3055	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3056	{
3057	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3058	return VINF_SUCCESS;
3059
3060	#ifdef VBOX_STRICT
3061	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3062	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3063	{
3064	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3065	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3066	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3067	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3068	}
3069	#endif
3070
3071	int rc = VERR_INTERNAL_ERROR_5;
3072	PVM pVM = pVCpu->CTX_SUFF(pVM);
3073	bool fInterceptDB = false;
3074	bool fInterceptMovDRx = false;
3075	if (DBGFIsStepping(pVCpu))
3076	{
3077	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3078	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3079	{
3080	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3081	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3082	AssertRCReturn(rc, rc);
3083	Assert(fInterceptDB == false);
3084	}
3085	else
3086	fInterceptDB = true;
3087	}
3088
3089	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3090	{
3091	if (!CPUMIsHyperDebugStateActive(pVCpu))
3092	{
3093	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3094	AssertRC(rc);
3095	}
3096	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3097	fInterceptMovDRx = true;
3098	}
3099	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3100	{
3101	if (!CPUMIsGuestDebugStateActive(pVCpu))
3102	{
3103	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3104	AssertRC(rc);
3105	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3106	}
3107	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3108	Assert(fInterceptMovDRx == false);
3109	}
3110	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3111	{
3112	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3113	fInterceptMovDRx = true;
3114	}
3115
3116	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3117	if (fInterceptDB)
3118	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3119	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3120	{
3121	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3122	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3123	#endif
3124	}
3125
3126	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3127	if (fInterceptMovDRx)
3128	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3129	else
3130	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3131
3132	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3133	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3134
3135	/* The guest's view of its DR7 is unblemished. */
3136	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3137
3138	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3139	return rc;
3140	}
3141
3142
3143	#ifdef VBOX_STRICT
3144	/**
3145	* Strict function to validate segment registers.
3146	*
3147	* @remarks Requires CR0.
3148	*/
3149	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3150	{
3151	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3152	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3153	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3154
3155	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3156	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3157	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3158	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3159	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3160	{
3161	/* Protected mode checks */
3162	/* CS */
3163	Assert(pCtx->cs.Attr.n.u1Present);
3164	Assert(!(pCtx->cs.Attr.u & 0xf00));
3165	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3166	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3167	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3168	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3169	\|\| (pCtx->cs.Attr.n.u1Granularity));
3170	/* CS cannot be loaded with NULL in protected mode. */
3171	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3172	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3173	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3174	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3175	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3176	else
3177	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3178	/* SS */
3179	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3180	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3181	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3182	if ( !(pCtx->cr0 & X86_CR0_PE)
3183	\|\| pCtx->cs.Attr.n.u4Type == 3)
3184	{
3185	Assert(!pCtx->ss.Attr.n.u2Dpl);
3186	}
3187	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3188	{
3189	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3190	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3191	Assert(pCtx->ss.Attr.n.u1Present);
3192	Assert(!(pCtx->ss.Attr.u & 0xf00));
3193	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3194	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3195	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3196	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3197	\|\| (pCtx->ss.Attr.n.u1Granularity));
3198	}
3199	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3200	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3201	{
3202	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3203	Assert(pCtx->ds.Attr.n.u1Present);
3204	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3205	Assert(!(pCtx->ds.Attr.u & 0xf00));
3206	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3207	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3208	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3209	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3210	\|\| (pCtx->ds.Attr.n.u1Granularity));
3211	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3212	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3213	}
3214	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3215	{
3216	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3217	Assert(pCtx->es.Attr.n.u1Present);
3218	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3219	Assert(!(pCtx->es.Attr.u & 0xf00));
3220	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3221	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3222	\|\| !(pCtx->es.Attr.n.u1Granularity));
3223	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3224	\|\| (pCtx->es.Attr.n.u1Granularity));
3225	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3226	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3227	}
3228	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3229	{
3230	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3231	Assert(pCtx->fs.Attr.n.u1Present);
3232	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3233	Assert(!(pCtx->fs.Attr.u & 0xf00));
3234	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3235	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3236	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3237	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3238	\|\| (pCtx->fs.Attr.n.u1Granularity));
3239	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3240	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3241	}
3242	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3243	{
3244	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3245	Assert(pCtx->gs.Attr.n.u1Present);
3246	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3247	Assert(!(pCtx->gs.Attr.u & 0xf00));
3248	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3249	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3250	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3251	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3252	\|\| (pCtx->gs.Attr.n.u1Granularity));
3253	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3254	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3255	}
3256	/* 64-bit capable CPUs. */
3257	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3258	Assert(!(pCtx->cs.u64Base >> 32));
3259	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3260	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3261	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3262	# endif
3263	}
3264	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3265	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3266	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3267	{
3268	/* Real and v86 mode checks. */
3269	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3270	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3271	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3272	{
3273	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3274	}
3275	else
3276	{
3277	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3278	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3279	}
3280
3281	/* CS */
3282	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3283	Assert(pCtx->cs.u32Limit == 0xffff);
3284	Assert(u32CSAttr == 0xf3);
3285	/* SS */
3286	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3287	Assert(pCtx->ss.u32Limit == 0xffff);
3288	Assert(u32SSAttr == 0xf3);
3289	/* DS */
3290	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3291	Assert(pCtx->ds.u32Limit == 0xffff);
3292	Assert(u32DSAttr == 0xf3);
3293	/* ES */
3294	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3295	Assert(pCtx->es.u32Limit == 0xffff);
3296	Assert(u32ESAttr == 0xf3);
3297	/* FS */
3298	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3299	Assert(pCtx->fs.u32Limit == 0xffff);
3300	Assert(u32FSAttr == 0xf3);
3301	/* GS */
3302	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3303	Assert(pCtx->gs.u32Limit == 0xffff);
3304	Assert(u32GSAttr == 0xf3);
3305	/* 64-bit capable CPUs. */
3306	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3307	Assert(!(pCtx->cs.u64Base >> 32));
3308	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3309	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3310	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3311	# endif
3312	}
3313	}
3314	#endif /* VBOX_STRICT */
3315
3316
3317	/**
3318	* Writes a guest segment register into the guest-state area in the VMCS.
3319	*
3320	* @returns VBox status code.
3321	* @param pVCpu Pointer to the VMCPU.
3322	* @param idxSel Index of the selector in the VMCS.
3323	* @param idxLimit Index of the segment limit in the VMCS.
3324	* @param idxBase Index of the segment base in the VMCS.
3325	* @param idxAccess Index of the access rights of the segment in the VMCS.
3326	* @param pSelReg Pointer to the segment selector.
3327	* @param pCtx Pointer to the guest-CPU context.
3328	*
3329	* @remarks No-long-jump zone!!!
3330	*/
3331	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3332	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3333	{
3334	int rc;
3335	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3336	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3337	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3338	AssertRCReturn(rc, rc);
3339
3340	uint32_t u32Access = pSelReg->Attr.u;
3341	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3342	{
3343	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3344	u32Access = 0xf3;
3345	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3346	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3347	}
3348	else
3349	{
3350	/*
3351	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3352	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3353	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3354	* loaded in protected-mode have their attribute as 0.
3355	*/
3356	if (!u32Access)
3357	u32Access = HMVMX_SEL_UNUSABLE;
3358	}
3359
3360	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3361	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3362	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3363
3364	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3365	AssertRCReturn(rc, rc);
3366	return rc;
3367	}
3368
3369
3370	/**
3371	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3372	* into the guest-state area in the VMCS.
3373	*
3374	* @returns VBox status code.
3375	* @param pVM Pointer to the VM.
3376	* @param pVCPU Pointer to the VMCPU.
3377	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3378	* out-of-sync. Make sure to update the required fields
3379	* before using them.
3380	*
3381	* @remarks Requires CR0 (strict builds validation).
3382	* @remarks No-long-jump zone!!!
3383	*/
3384	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3385	{
3386	int rc = VERR_INTERNAL_ERROR_5;
3387	PVM pVM = pVCpu->CTX_SUFF(pVM);
3388
3389	/*
3390	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3391	*/
3392	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3393	{
3394	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3395	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3396	{
3397	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3398	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3399	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3400	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3401	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3402	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3403	}
3404
3405	#ifdef VBOX_WITH_REM
3406	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3407	{
3408	Assert(pVM->hm.s.vmx.pRealModeTSS);
3409	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3410	if ( pVCpu->hm.s.vmx.fWasInRealMode
3411	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3412	{
3413	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3414	in real-mode (e.g. OpenBSD 4.0) */
3415	REMFlushTBs(pVM);
3416	Log(("Load: Switch to protected mode detected!\n"));
3417	pVCpu->hm.s.vmx.fWasInRealMode = false;
3418	}
3419	}
3420	#endif
3421	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3422	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3423	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3424	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3425	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3426	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3427	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3428	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3429	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3430	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3431	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3432	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3433	AssertRCReturn(rc, rc);
3434
3435	#ifdef VBOX_STRICT
3436	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3437	#endif
3438	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3439	}
3440
3441	/*
3442	* Guest TR.
3443	*/
3444	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3445	{
3446	/*
3447	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3448	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3449	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3450	*/
3451	uint16_t u16Sel = 0;
3452	uint32_t u32Limit = 0;
3453	uint64_t u64Base = 0;
3454	uint32_t u32AccessRights = 0;
3455
3456	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3457	{
3458	u16Sel = pMixedCtx->tr.Sel;
3459	u32Limit = pMixedCtx->tr.u32Limit;
3460	u64Base = pMixedCtx->tr.u64Base;
3461	u32AccessRights = pMixedCtx->tr.Attr.u;
3462	}
3463	else
3464	{
3465	Assert(pVM->hm.s.vmx.pRealModeTSS);
3466	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3467
3468	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3469	RTGCPHYS GCPhys;
3470	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3471	AssertRCReturn(rc, rc);
3472
3473	X86DESCATTR DescAttr;
3474	DescAttr.u = 0;
3475	DescAttr.n.u1Present = 1;
3476	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3477
3478	u16Sel = 0;
3479	u32Limit = HM_VTX_TSS_SIZE;
3480	u64Base = GCPhys; /* in real-mode phys = virt. */
3481	u32AccessRights = DescAttr.u;
3482	}
3483
3484	/* Validate. */
3485	Assert(!(u16Sel & RT_BIT(2)));
3486	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3487	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3488	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3489	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3490	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3491	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3492	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3493	Assert( (u32Limit & 0xfff) == 0xfff
3494	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3495	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3496	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3497
3498	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3499	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3500	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3501	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3502	AssertRCReturn(rc, rc);
3503
3504	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3505	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3506	}
3507
3508	/*
3509	* Guest GDTR.
3510	*/
3511	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3512	{
3513	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3514	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3515	AssertRCReturn(rc, rc);
3516
3517	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3518	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3519	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3520	}
3521
3522	/*
3523	* Guest LDTR.
3524	*/
3525	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3526	{
3527	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3528	uint32_t u32Access = 0;
3529	if (!pMixedCtx->ldtr.Attr.u)
3530	u32Access = HMVMX_SEL_UNUSABLE;
3531	else
3532	u32Access = pMixedCtx->ldtr.Attr.u;
3533
3534	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3535	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3536	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3537	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3538	AssertRCReturn(rc, rc);
3539
3540	/* Validate. */
3541	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3542	{
3543	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3544	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3545	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3546	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3547	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3548	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3549	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3550	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3551	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3552	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3553	}
3554
3555	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3556	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3557	}
3558
3559	/*
3560	* Guest IDTR.
3561	*/
3562	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3563	{
3564	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3565	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3566	AssertRCReturn(rc, rc);
3567
3568	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3569	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3570	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3571	}
3572
3573	return VINF_SUCCESS;
3574	}
3575
3576
3577	/**
3578	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3579	* areas. These MSRs will automatically be loaded to the host CPU on every
3580	* successful VM entry and stored from the host CPU on every successful VM exit.
3581	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3582	*
3583	* @returns VBox status code.
3584	* @param pVCpu Pointer to the VMCPU.
3585	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3586	* out-of-sync. Make sure to update the required fields
3587	* before using them.
3588	*
3589	* @remarks No-long-jump zone!!!
3590	*/
3591	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3592	{
3593	AssertPtr(pVCpu);
3594	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3595
3596	/*
3597	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3598	*/
3599	int rc = VINF_SUCCESS;
3600	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3601	{
3602	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3603	PVM pVM = pVCpu->CTX_SUFF(pVM);
3604	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3605	uint32_t cGuestMsrs = 0;
3606
3607	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3608	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3609	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3610	if (fSupportsNX \|\| fSupportsLongMode)
3611	{
3612	/** @todo support save IA32_EFER, i.e.
3613	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3614	* guest EFER need not be part of the VM-entry MSR-load area. */
3615	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3616	pGuestMsr->u32Reserved = 0;
3617	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3618	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3619	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3620	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3621	pGuestMsr++; cGuestMsrs++;
3622	if (fSupportsLongMode)
3623	{
3624	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3625	pGuestMsr->u32Reserved = 0;
3626	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3627	pGuestMsr++; cGuestMsrs++;
3628	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3629	pGuestMsr->u32Reserved = 0;
3630	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3631	pGuestMsr++; cGuestMsrs++;
3632	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3633	pGuestMsr->u32Reserved = 0;
3634	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3635	pGuestMsr++; cGuestMsrs++;
3636	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3637	pGuestMsr->u32Reserved = 0;
3638	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3639	pGuestMsr++; cGuestMsrs++;
3640	}
3641	}
3642
3643	/*
3644	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3645	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3646	*/
3647	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3648	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3649	{
3650	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3651	pGuestMsr->u32Reserved = 0;
3652	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3653	AssertRCReturn(rc, rc);
3654	pGuestMsr++; cGuestMsrs++;
3655	}
3656
3657	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3658	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3659	{
3660	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3661	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3662	}
3663
3664	/* Update the VCPU's copy of the guest MSR count. */
3665	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3666	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3667	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3668	AssertRCReturn(rc, rc);
3669	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3670
3671	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3672	}
3673
3674	/*
3675	* Guest Sysenter MSRs.
3676	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3677	* VM-exits on WRMSRs for these MSRs.
3678	*/
3679	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3680	{
3681	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3682	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3683	}
3684	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3685	{
3686	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3687	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3688	}
3689	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3690	{
3691	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3692	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3693	}
3694	AssertRCReturn(rc, rc);
3695
3696	return rc;
3697	}
3698
3699
3700	/**
3701	* Loads the guest activity state into the guest-state area in the VMCS.
3702	*
3703	* @returns VBox status code.
3704	* @param pVCpu Pointer to the VMCPU.
3705	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3706	* out-of-sync. Make sure to update the required fields
3707	* before using them.
3708	*
3709	* @remarks No-long-jump zone!!!
3710	*/
3711	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3712	{
3713	/** @todo See if we can make use of other states, e.g.
3714	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3715	int rc = VINF_SUCCESS;
3716	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3717	{
3718	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3719	AssertRCReturn(rc, rc);
3720	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3721	}
3722	return rc;
3723	}
3724
3725
3726	/**
3727	* Sets up the appropriate function to run guest code.
3728	*
3729	* @returns VBox status code.
3730	* @param pVCpu Pointer to the VMCPU.
3731	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3732	* out-of-sync. Make sure to update the required fields
3733	* before using them.
3734	*
3735	* @remarks No-long-jump zone!!!
3736	*/
3737	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3738	{
3739	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3740	{
3741	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3742	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3743	#endif
3744	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3745	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3746	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3747	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3748	#else
3749	/* 64-bit host or hybrid host. */
3750	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3751	#endif
3752	}
3753	else
3754	{
3755	/* Guest is not in long mode, use the 32-bit handler. */
3756	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3757	}
3758	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3759	return VINF_SUCCESS;
3760	}
3761
3762
3763	/**
3764	* Wrapper for running the guest code in VT-x.
3765	*
3766	* @returns VBox strict status code.
3767	* @param pVM Pointer to the VM.
3768	* @param pVCpu Pointer to the VMCPU.
3769	* @param pCtx Pointer to the guest-CPU context.
3770	*
3771	* @remarks No-long-jump zone!!!
3772	*/
3773	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3774	{
3775	/*
3776	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3777	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3778	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3779	*/
3780	#ifdef VBOX_WITH_KERNEL_USING_XMM
3781	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3782	#else
3783	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3784	#endif
3785	}
3786
3787
3788	/**
3789	* Report world-switch error and dump some useful debug info.
3790	*
3791	* @param pVM Pointer to the VM.
3792	* @param pVCpu Pointer to the VMCPU.
3793	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3794	* @param pCtx Pointer to the guest-CPU context.
3795	* @param pVmxTransient Pointer to the VMX transient structure (only
3796	* exitReason updated).
3797	*/
3798	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3799	{
3800	Assert(pVM);
3801	Assert(pVCpu);
3802	Assert(pCtx);
3803	Assert(pVmxTransient);
3804	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3805
3806	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3807	switch (rcVMRun)
3808	{
3809	case VERR_VMX_INVALID_VMXON_PTR:
3810	AssertFailed();
3811	break;
3812	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3813	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3814	{
3815	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3816	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3817	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3818	AssertRC(rc);
3819
3820	#ifdef VBOX_STRICT
3821	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3822	pVmxTransient->uExitReason));
3823	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3824	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3825	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3826	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3827	else
3828	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3829
3830	/* VMX control bits. */
3831	uint32_t u32Val;
3832	uint64_t u64Val;
3833	HMVMXHCUINTREG uHCReg;
3834	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3835	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3836	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3837	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3838	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3839	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3840	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3841	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3842	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3843	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3844	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3845	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3846	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3847	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3848	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3849	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3850	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3851	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3852	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3853	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3854	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3855	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3856	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3857	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3858	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3859	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3860	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3861	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3862	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3863	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3864	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3865	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3866	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3867	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3868	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3869	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3870	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3871	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3872	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3873	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3874	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3875	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3876
3877	/* Guest bits. */
3878	RTGCUINTREG uGCReg;
3879	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3880	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3881	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3882	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3883	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3884	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3885	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3886	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3887
3888	/* Host bits. */
3889	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3890	Log(("Host CR0 %#RHr\n", uHCReg));
3891	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3892	Log(("Host CR3 %#RHr\n", uHCReg));
3893	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3894	Log(("Host CR4 %#RHr\n", uHCReg));
3895
3896	RTGDTR HostGdtr;
3897	PCX86DESCHC pDesc;
3898	ASMGetGDTR(&HostGdtr);
3899	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3900	Log(("Host CS %#08x\n", u32Val));
3901	if (u32Val < HostGdtr.cbGdt)
3902	{
3903	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3904	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3905	}
3906
3907	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3908	Log(("Host DS %#08x\n", u32Val));
3909	if (u32Val < HostGdtr.cbGdt)
3910	{
3911	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3912	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3913	}
3914
3915	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3916	Log(("Host ES %#08x\n", u32Val));
3917	if (u32Val < HostGdtr.cbGdt)
3918	{
3919	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3920	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3921	}
3922
3923	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3924	Log(("Host FS %#08x\n", u32Val));
3925	if (u32Val < HostGdtr.cbGdt)
3926	{
3927	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3928	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3929	}
3930
3931	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3932	Log(("Host GS %#08x\n", u32Val));
3933	if (u32Val < HostGdtr.cbGdt)
3934	{
3935	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3936	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3937	}
3938
3939	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3940	Log(("Host SS %#08x\n", u32Val));
3941	if (u32Val < HostGdtr.cbGdt)
3942	{
3943	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3944	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3945	}
3946
3947	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3948	Log(("Host TR %#08x\n", u32Val));
3949	if (u32Val < HostGdtr.cbGdt)
3950	{
3951	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3952	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3953	}
3954
3955	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3956	Log(("Host TR Base %#RHv\n", uHCReg));
3957	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3958	Log(("Host GDTR Base %#RHv\n", uHCReg));
3959	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3960	Log(("Host IDTR Base %#RHv\n", uHCReg));
3961	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3962	Log(("Host SYSENTER CS %#08x\n", u32Val));
3963	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3964	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3965	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3966	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3967	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3968	Log(("Host RSP %#RHv\n", uHCReg));
3969	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3970	Log(("Host RIP %#RHv\n", uHCReg));
3971	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3972	if (HMVMX_IS_64BIT_HOST_MODE())
3973	{
3974	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3975	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3976	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3977	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3978	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3979	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3980	}
3981	# endif
3982	#endif /* VBOX_STRICT */
3983	break;
3984	}
3985
3986	default:
3987	/* Impossible */
3988	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3989	break;
3990	}
3991	NOREF(pVM);
3992	}
3993
3994
3995	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3996	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3997	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3998	#endif
3999	#ifdef VBOX_STRICT
4000	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4001	{
4002	switch (idxField)
4003	{
4004	case VMX_VMCS_GUEST_RIP:
4005	case VMX_VMCS_GUEST_RSP:
4006	case VMX_VMCS_GUEST_DR7:
4007	case VMX_VMCS_GUEST_SYSENTER_EIP:
4008	case VMX_VMCS_GUEST_SYSENTER_ESP:
4009	case VMX_VMCS_GUEST_GDTR_BASE:
4010	case VMX_VMCS_GUEST_IDTR_BASE:
4011	case VMX_VMCS_GUEST_CS_BASE:
4012	case VMX_VMCS_GUEST_DS_BASE:
4013	case VMX_VMCS_GUEST_ES_BASE:
4014	case VMX_VMCS_GUEST_FS_BASE:
4015	case VMX_VMCS_GUEST_GS_BASE:
4016	case VMX_VMCS_GUEST_SS_BASE:
4017	case VMX_VMCS_GUEST_LDTR_BASE:
4018	case VMX_VMCS_GUEST_TR_BASE:
4019	case VMX_VMCS_GUEST_CR3:
4020	return true;
4021	}
4022	return false;
4023	}
4024
4025	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4026	{
4027	switch (idxField)
4028	{
4029	/* Read-only fields. */
4030	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4031	return true;
4032	}
4033	/* Remaining readable fields should also be writable. */
4034	return hmR0VmxIsValidWriteField(idxField);
4035	}
4036	#endif /* VBOX_STRICT */
4037
4038	/**
4039	* Executes the specified handler in 64-bit mode.
4040	*
4041	* @returns VBox status code.
4042	* @param pVM Pointer to the VM.
4043	* @param pVCpu Pointer to the VMCPU.
4044	* @param pCtx Pointer to the guest CPU context.
4045	* @param enmOp The operation to perform.
4046	* @param cbParam Number of parameters.
4047	* @param paParam Array of 32-bit parameters.
4048	*/
4049	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4050	uint32_t *paParam)
4051	{
4052	int rc, rc2;
4053	PHMGLOBLCPUINFO pCpu;
4054	RTHCPHYS HCPhysCpuPage;
4055	RTCCUINTREG uOldEFlags;
4056
4057	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4058	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4059	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4060	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4061
4062	#ifdef VBOX_STRICT
4063	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4064	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4065
4066	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4067	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4068	#endif
4069
4070	/* Disable interrupts. */
4071	uOldEFlags = ASMIntDisableFlags();
4072
4073	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4074	RTCPUID idHostCpu = RTMpCpuId();
4075	CPUMR0SetLApic(pVM, idHostCpu);
4076	#endif
4077
4078	pCpu = HMR0GetCurrentCpu();
4079	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4080
4081	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4082	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4083
4084	/* Leave VMX Root Mode. */
4085	VMXDisable();
4086
4087	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4088
4089	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4090	CPUMSetHyperEIP(pVCpu, enmOp);
4091	for (int i = (int)cbParam - 1; i >= 0; i--)
4092	CPUMPushHyper(pVCpu, paParam[i]);
4093
4094	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4095
4096	/* Call the switcher. */
4097	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4098	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4099
4100	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4101	/* Make sure the VMX instructions don't cause #UD faults. */
4102	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4103
4104	/* Re-enter VMX Root Mode */
4105	rc2 = VMXEnable(HCPhysCpuPage);
4106	if (RT_FAILURE(rc2))
4107	{
4108	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4109	ASMSetFlags(uOldEFlags);
4110	return rc2;
4111	}
4112
4113	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4114	AssertRC(rc2);
4115	Assert(!(ASMGetFlags() & X86_EFL_IF));
4116	ASMSetFlags(uOldEFlags);
4117	return rc;
4118	}
4119
4120
4121	/**
4122	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4123	* supporting 64-bit guests.
4124	*
4125	* @returns VBox status code.
4126	* @param fResume Whether to VMLAUNCH or VMRESUME.
4127	* @param pCtx Pointer to the guest-CPU context.
4128	* @param pCache Pointer to the VMCS cache.
4129	* @param pVM Pointer to the VM.
4130	* @param pVCpu Pointer to the VMCPU.
4131	*/
4132	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4133	{
4134	uint32_t aParam[6];
4135	PHMGLOBLCPUINFO pCpu = NULL;
4136	RTHCPHYS HCPhysCpuPage = 0;
4137	int rc = VERR_INTERNAL_ERROR_5;
4138
4139	pCpu = HMR0GetCurrentCpu();
4140	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4141
4142	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4143	pCache->uPos = 1;
4144	pCache->interPD = PGMGetInterPaeCR3(pVM);
4145	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4146	#endif
4147
4148	#ifdef VBOX_STRICT
4149	pCache->TestIn.HCPhysCpuPage = 0;
4150	pCache->TestIn.HCPhysVmcs = 0;
4151	pCache->TestIn.pCache = 0;
4152	pCache->TestOut.HCPhysVmcs = 0;
4153	pCache->TestOut.pCache = 0;
4154	pCache->TestOut.pCtx = 0;
4155	pCache->TestOut.eflags = 0;
4156	#endif
4157
4158	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4159	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4160	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4161	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4162	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4163	aParam[5] = 0;
4164
4165	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4166	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4167	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4168	#endif
4169	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4170
4171	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4172	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4173	Assert(pCtx->dr[4] == 10);
4174	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4175	#endif
4176
4177	#ifdef VBOX_STRICT
4178	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4179	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4180	pVCpu->hm.s.vmx.HCPhysVmcs));
4181	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4182	pCache->TestOut.HCPhysVmcs));
4183	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4184	pCache->TestOut.pCache));
4185	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4186	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4187	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4188	pCache->TestOut.pCtx));
4189	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4190	#endif
4191	return rc;
4192	}
4193
4194
4195	/**
4196	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4197	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4198	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4199	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4200	*
4201	* @returns VBox status code.
4202	* @param pVM Pointer to the VM.
4203	* @param pVCpu Pointer to the VMCPU.
4204	*/
4205	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4206	{
4207	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4208	{ \
4209	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4210	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4211	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4212	++cReadFields; \
4213	}
4214
4215	AssertPtr(pVM);
4216	AssertPtr(pVCpu);
4217	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4218	uint32_t cReadFields = 0;
4219
4220	/* Guest-natural selector base fields */
4221	#if 0
4222	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4223	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4224	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4225	#endif
4226	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4227	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4228	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4229	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4230	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4231	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4232	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4239	#if 0
4240	/* Unused natural width guest-state fields. */
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4243	#endif
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4245	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4246
4247	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4248	#if 0
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4252	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4257	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4258	#endif
4259
4260	/* Natural width guest-state fields. */
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4262	#if 0
4263	/* Currently unused field. */
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4265	#endif
4266
4267	if (pVM->hm.s.fNestedPaging)
4268	{
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4270	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4271	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4272	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4273	}
4274	else
4275	{
4276	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4277	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4278	}
4279
4280	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4281	return VINF_SUCCESS;
4282	}
4283
4284
4285	/**
4286	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4287	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4288	* darwin, running 64-bit guests).
4289	*
4290	* @returns VBox status code.
4291	* @param pVCpu Pointer to the VMCPU.
4292	* @param idxField The VMCS field encoding.
4293	* @param u64Val 16, 32 or 64 bits value.
4294	*/
4295	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4296	{
4297	int rc;
4298	switch (idxField)
4299	{
4300	/*
4301	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4302	*/
4303	/* 64-bit Control fields. */
4304	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4305	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4306	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4307	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4308	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4309	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4310	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4311	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4312	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4313	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4314	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4315	case VMX_VMCS64_CTRL_EPTP_FULL:
4316	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4317	/* 64-bit Guest-state fields. */
4318	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4319	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4320	case VMX_VMCS64_GUEST_PAT_FULL:
4321	case VMX_VMCS64_GUEST_EFER_FULL:
4322	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4323	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4324	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4325	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4326	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4327	/* 64-bit Host-state fields. */
4328	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4329	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4330	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4331	{
4332	rc = VMXWriteVmcs32(idxField, u64Val);
4333	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4334	break;
4335	}
4336
4337	/*
4338	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4339	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4340	*/
4341	/* Natural-width Guest-state fields. */
4342	case VMX_VMCS_GUEST_CR3:
4343	case VMX_VMCS_GUEST_ES_BASE:
4344	case VMX_VMCS_GUEST_CS_BASE:
4345	case VMX_VMCS_GUEST_SS_BASE:
4346	case VMX_VMCS_GUEST_DS_BASE:
4347	case VMX_VMCS_GUEST_FS_BASE:
4348	case VMX_VMCS_GUEST_GS_BASE:
4349	case VMX_VMCS_GUEST_LDTR_BASE:
4350	case VMX_VMCS_GUEST_TR_BASE:
4351	case VMX_VMCS_GUEST_GDTR_BASE:
4352	case VMX_VMCS_GUEST_IDTR_BASE:
4353	case VMX_VMCS_GUEST_DR7:
4354	case VMX_VMCS_GUEST_RSP:
4355	case VMX_VMCS_GUEST_RIP:
4356	case VMX_VMCS_GUEST_SYSENTER_ESP:
4357	case VMX_VMCS_GUEST_SYSENTER_EIP:
4358	{
4359	if (!(u64Val >> 32))
4360	{
4361	/* If this field is 64-bit, VT-x will zero out the top bits. */
4362	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4363	}
4364	else
4365	{
4366	/* Assert that only the 32->64 switcher case should ever come here. */
4367	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4368	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4369	}
4370	break;
4371	}
4372
4373	default:
4374	{
4375	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4376	rc = VERR_INVALID_PARAMETER;
4377	break;
4378	}
4379	}
4380	AssertRCReturn(rc, rc);
4381	return rc;
4382	}
4383
4384
4385	/**
4386	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4387	* hosts (except darwin) for 64-bit guests.
4388	*
4389	* @param pVCpu Pointer to the VMCPU.
4390	* @param idxField The VMCS field encoding.
4391	* @param u64Val 16, 32 or 64 bits value.
4392	*/
4393	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4394	{
4395	AssertPtr(pVCpu);
4396	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4397
4398	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4399	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4400
4401	/* Make sure there are no duplicates. */
4402	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4403	{
4404	if (pCache->Write.aField[i] == idxField)
4405	{
4406	pCache->Write.aFieldVal[i] = u64Val;
4407	return VINF_SUCCESS;
4408	}
4409	}
4410
4411	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4412	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4413	pCache->Write.cValidEntries++;
4414	return VINF_SUCCESS;
4415	}
4416
4417	/* Enable later when the assembly code uses these as callbacks. */
4418	#if 0
4419	/*
4420	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4421	*
4422	* @param pVCpu Pointer to the VMCPU.
4423	* @param pCache Pointer to the VMCS cache.
4424	*
4425	* @remarks No-long-jump zone!!!
4426	*/
4427	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4428	{
4429	AssertPtr(pCache);
4430	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4431	{
4432	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4433	AssertRC(rc);
4434	}
4435	pCache->Write.cValidEntries = 0;
4436	}
4437
4438
4439	/**
4440	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4441	*
4442	* @param pVCpu Pointer to the VMCPU.
4443	* @param pCache Pointer to the VMCS cache.
4444	*
4445	* @remarks No-long-jump zone!!!
4446	*/
4447	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4448	{
4449	AssertPtr(pCache);
4450	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4451	{
4452	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4453	AssertRC(rc);
4454	}
4455	}
4456	#endif
4457	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4458
4459
4460	/**
4461	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4462	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4463	* timer.
4464	*
4465	* @returns VBox status code.
4466	* @param pVCpu Pointer to the VMCPU.
4467	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4468	* out-of-sync. Make sure to update the required fields
4469	* before using them.
4470	* @remarks No-long-jump zone!!!
4471	*/
4472	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4473	{
4474	int rc = VERR_INTERNAL_ERROR_5;
4475	bool fOffsettedTsc = false;
4476	PVM pVM = pVCpu->CTX_SUFF(pVM);
4477	if (pVM->hm.s.vmx.fUsePreemptTimer)
4478	{
4479	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4480
4481	/* Make sure the returned values have sane upper and lower boundaries. */
4482	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4483	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4484	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4485	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4486
4487	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4488	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4489	}
4490	else
4491	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4492
4493	if (fOffsettedTsc)
4494	{
4495	uint64_t u64CurTSC = ASMReadTSC();
4496	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4497	{
4498	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4499	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4500
4501	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4502	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4503	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4504	}
4505	else
4506	{
4507	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4508	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4509	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4510	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4511	}
4512	}
4513	else
4514	{
4515	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4516	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4517	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4518	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4519	}
4520	}
4521
4522
4523	/**
4524	* Determines if an exception is a contributory exception. Contributory
4525	* exceptions are ones which can cause double-faults. Page-fault is
4526	* intentionally not included here as it's a conditional contributory exception.
4527	*
4528	* @returns true if the exception is contributory, false otherwise.
4529	* @param uVector The exception vector.
4530	*/
4531	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4532	{
4533	switch (uVector)
4534	{
4535	case X86_XCPT_GP:
4536	case X86_XCPT_SS:
4537	case X86_XCPT_NP:
4538	case X86_XCPT_TS:
4539	case X86_XCPT_DE:
4540	return true;
4541	default:
4542	break;
4543	}
4544	return false;
4545	}
4546
4547
4548	/**
4549	* Sets an event as a pending event to be injected into the guest.
4550	*
4551	* @param pVCpu Pointer to the VMCPU.
4552	* @param u32IntrInfo The VM-entry interruption-information field.
4553	* @param cbInstr The VM-entry instruction length in bytes (for software
4554	* interrupts, exceptions and privileged software
4555	* exceptions).
4556	* @param u32ErrCode The VM-entry exception error code.
4557	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4558	* page-fault.
4559	*/
4560	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4561	RTGCUINTPTR GCPtrFaultAddress)
4562	{
4563	Assert(!pVCpu->hm.s.Event.fPending);
4564	pVCpu->hm.s.Event.fPending = true;
4565	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4566	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4567	pVCpu->hm.s.Event.cbInstr = cbInstr;
4568	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4569	}
4570
4571
4572	/**
4573	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4574	*
4575	* @param pVCpu Pointer to the VMCPU.
4576	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4577	* out-of-sync. Make sure to update the required fields
4578	* before using them.
4579	*/
4580	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4581	{
4582	/* Inject the double-fault. */
4583	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4584	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4585	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4586	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4587	}
4588
4589
4590	/**
4591	* Handle a condition that occurred while delivering an event through the guest
4592	* IDT.
4593	*
4594	* @returns VBox status code (informational error codes included).
4595	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4596	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4597	* continue execution of the guest which will delivery the #DF.
4598	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4599	*
4600	* @param pVCpu Pointer to the VMCPU.
4601	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4602	* out-of-sync. Make sure to update the required fields
4603	* before using them.
4604	* @param pVmxTransient Pointer to the VMX transient structure.
4605	*
4606	* @remarks No-long-jump zone!!!
4607	*/
4608	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4609	{
4610	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4611	AssertRC(rc);
4612	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4613	{
4614	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4615	AssertRCReturn(rc, rc);
4616
4617	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4618	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4619	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4620
4621	typedef enum
4622	{
4623	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4624	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4625	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4626	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4627	} VMXREFLECTXCPT;
4628
4629	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4630	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4631	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4632	{
4633	enmReflect = VMXREFLECTXCPT_XCPT;
4634	#ifdef VBOX_STRICT
4635	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4636	&& uExitVector == X86_XCPT_PF)
4637	{
4638	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4639	}
4640	#endif
4641	if ( uExitVector == X86_XCPT_PF
4642	&& uIdtVector == X86_XCPT_PF)
4643	{
4644	pVmxTransient->fVectoringPF = true;
4645	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4646	}
4647	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4648	&& hmR0VmxIsContributoryXcpt(uExitVector)
4649	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4650	\|\| uIdtVector == X86_XCPT_PF))
4651	{
4652	enmReflect = VMXREFLECTXCPT_DF;
4653	}
4654	else if (uIdtVector == X86_XCPT_DF)
4655	enmReflect = VMXREFLECTXCPT_TF;
4656	}
4657	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4658	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4659	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4660	{
4661	/*
4662	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4663	* (whatever they are) as they reoccur when restarting the instruction.
4664	*/
4665	enmReflect = VMXREFLECTXCPT_XCPT;
4666	}
4667
4668	switch (enmReflect)
4669	{
4670	case VMXREFLECTXCPT_XCPT:
4671	{
4672	uint32_t u32ErrCode = 0;
4673	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4674	{
4675	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4676	AssertRCReturn(rc, rc);
4677	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4678	}
4679
4680	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4681	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4682	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4683	rc = VINF_SUCCESS;
4684	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4685	pVCpu->hm.s.Event.u32ErrCode));
4686	break;
4687	}
4688
4689	case VMXREFLECTXCPT_DF:
4690	{
4691	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4692	rc = VINF_VMX_DOUBLE_FAULT;
4693	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4694	uIdtVector, uExitVector));
4695	break;
4696	}
4697
4698	case VMXREFLECTXCPT_TF:
4699	{
4700	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4701	rc = VINF_EM_RESET;
4702	break;
4703	}
4704
4705	default:
4706	Assert(rc == VINF_SUCCESS);
4707	break;
4708	}
4709	}
4710	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4711	return rc;
4712	}
4713
4714
4715	/**
4716	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4717	*
4718	* @returns VBox status code.
4719	* @param pVCpu Pointer to the VMCPU.
4720	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4721	* out-of-sync. Make sure to update the required fields
4722	* before using them.
4723	*
4724	* @remarks No-long-jump zone!!!
4725	*/
4726	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4727	{
4728	int rc = VINF_SUCCESS;
4729	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4730	{
4731	uint32_t uVal = 0;
4732	uint32_t uShadow = 0;
4733	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4734	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4735	AssertRCReturn(rc, rc);
4736	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4737	CPUMSetGuestCR0(pVCpu, uVal);
4738	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4739	}
4740	return rc;
4741	}
4742
4743
4744	/**
4745	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4746	*
4747	* @returns VBox status code.
4748	* @param pVCpu Pointer to the VMCPU.
4749	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4750	* out-of-sync. Make sure to update the required fields
4751	* before using them.
4752	*
4753	* @remarks No-long-jump zone!!!
4754	*/
4755	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4756	{
4757	int rc = VINF_SUCCESS;
4758	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4759	{
4760	uint32_t uVal = 0;
4761	uint32_t uShadow = 0;
4762	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4763	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4764	AssertRCReturn(rc, rc);
4765	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4766	CPUMSetGuestCR4(pVCpu, uVal);
4767	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4768	}
4769	return rc;
4770	}
4771
4772
4773	/**
4774	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4775	*
4776	* @returns VBox status code.
4777	* @param pVCpu Pointer to the VMCPU.
4778	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4779	* out-of-sync. Make sure to update the required fields
4780	* before using them.
4781	*
4782	* @remarks No-long-jump zone!!!
4783	*/
4784	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4785	{
4786	int rc = VINF_SUCCESS;
4787	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4788	{
4789	RTGCUINTREG uVal = 0;
4790	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4791	AssertRCReturn(rc, rc);
4792	pMixedCtx->rip = uVal;
4793	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4794	}
4795	return rc;
4796	}
4797
4798
4799	/**
4800	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4801	*
4802	* @returns VBox status code.
4803	* @param pVCpu Pointer to the VMCPU.
4804	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4805	* out-of-sync. Make sure to update the required fields
4806	* before using them.
4807	*
4808	* @remarks No-long-jump zone!!!
4809	*/
4810	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811	{
4812	int rc = VINF_SUCCESS;
4813	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4814	{
4815	RTGCUINTREG uVal = 0;
4816	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4817	AssertRCReturn(rc, rc);
4818	pMixedCtx->rsp = uVal;
4819	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4820	}
4821	return rc;
4822	}
4823
4824
4825	/**
4826	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4827	*
4828	* @returns VBox status code.
4829	* @param pVCpu Pointer to the VMCPU.
4830	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4831	* out-of-sync. Make sure to update the required fields
4832	* before using them.
4833	*
4834	* @remarks No-long-jump zone!!!
4835	*/
4836	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837	{
4838	int rc = VINF_SUCCESS;
4839	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4840	{
4841	uint32_t uVal = 0;
4842	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4843	AssertRCReturn(rc, rc);
4844	pMixedCtx->eflags.u32 = uVal;
4845
4846	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4847	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4848	{
4849	PVM pVM = pVCpu->CTX_SUFF(pVM);
4850	Assert(pVM->hm.s.vmx.pRealModeTSS);
4851	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4852	pMixedCtx->eflags.Bits.u1VM = 0;
4853	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4854	}
4855
4856	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4857	}
4858	return rc;
4859	}
4860
4861
4862	/**
4863	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4864	* guest-CPU context.
4865	*/
4866	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4867	{
4868	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4869	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4870	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4871	return rc;
4872	}
4873
4874
4875	/**
4876	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4877	* from the guest-state area in the VMCS.
4878	*
4879	* @param pVCpu Pointer to the VMCPU.
4880	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4881	* out-of-sync. Make sure to update the required fields
4882	* before using them.
4883	*
4884	* @remarks No-long-jump zone!!!
4885	*/
4886	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4887	{
4888	uint32_t uIntrState = 0;
4889	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4890	AssertRC(rc);
4891
4892	if (!uIntrState)
4893	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4894	else
4895	{
4896	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4897	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4898	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4899	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4900	AssertRC(rc);
4901	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4902	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4903	}
4904	}
4905
4906
4907	/**
4908	* Saves the guest's activity state.
4909	*
4910	* @returns VBox status code.
4911	* @param pVCpu Pointer to the VMCPU.
4912	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4913	* out-of-sync. Make sure to update the required fields
4914	* before using them.
4915	*
4916	* @remarks No-long-jump zone!!!
4917	*/
4918	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4919	{
4920	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4921	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4922	return VINF_SUCCESS;
4923	}
4924
4925
4926	/**
4927	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4928	* the current VMCS into the guest-CPU context.
4929	*
4930	* @returns VBox status code.
4931	* @param pVCpu Pointer to the VMCPU.
4932	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4933	* out-of-sync. Make sure to update the required fields
4934	* before using them.
4935	*
4936	* @remarks No-long-jump zone!!!
4937	*/
4938	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4939	{
4940	int rc = VINF_SUCCESS;
4941	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4942	{
4943	uint32_t u32Val = 0;
4944	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4945	pMixedCtx->SysEnter.cs = u32Val;
4946	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4947	}
4948
4949	RTGCUINTREG uGCVal = 0;
4950	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4951	{
4952	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4953	pMixedCtx->SysEnter.eip = uGCVal;
4954	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4955	}
4956	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4957	{
4958	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4959	pMixedCtx->SysEnter.esp = uGCVal;
4960	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4961	}
4962	return rc;
4963	}
4964
4965
4966	/**
4967	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4968	* context.
4969	*
4970	* @returns VBox status code.
4971	* @param pVCpu Pointer to the VMCPU.
4972	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4973	* out-of-sync. Make sure to update the required fields
4974	* before using them.
4975	*
4976	* @remarks No-long-jump zone!!!
4977	*/
4978	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4979	{
4980	int rc = VINF_SUCCESS;
4981	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4982	{
4983	RTGCUINTREG uVal = 0;
4984	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4985	pMixedCtx->fs.u64Base = uVal;
4986	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4987	}
4988	return rc;
4989	}
4990
4991
4992	/**
4993	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4994	* context.
4995	*
4996	* @returns VBox status code.
4997	* @param pVCpu Pointer to the VMCPU.
4998	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4999	* out-of-sync. Make sure to update the required fields
5000	* before using them.
5001	*
5002	* @remarks No-long-jump zone!!!
5003	*/
5004	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5005	{
5006	int rc = VINF_SUCCESS;
5007	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5008	{
5009	RTGCUINTREG uVal = 0;
5010	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5011	pMixedCtx->gs.u64Base = uVal;
5012	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5013	}
5014	return rc;
5015	}
5016
5017
5018	/**
5019	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5020	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5021	*
5022	* @returns VBox status code.
5023	* @param pVCpu Pointer to the VMCPU.
5024	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5025	* out-of-sync. Make sure to update the required fields
5026	* before using them.
5027	*
5028	* @remarks No-long-jump zone!!!
5029	*/
5030	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5031	{
5032	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5033	return VINF_SUCCESS;
5034
5035	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5036	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5037	{
5038	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5039	pMsr += i;
5040	switch (pMsr->u32IndexMSR)
5041	{
5042	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5043	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5044	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5045	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5046	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5047	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5048	default:
5049	{
5050	AssertFailed();
5051	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5052	}
5053	}
5054	}
5055	#endif
5056
5057	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5058	return VINF_SUCCESS;
5059	}
5060
5061
5062	/**
5063	* Saves the guest control registers from the current VMCS into the guest-CPU
5064	* context.
5065	*
5066	* @returns VBox status code.
5067	* @param pVCpu Pointer to the VMCPU.
5068	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5069	* out-of-sync. Make sure to update the required fields
5070	* before using them.
5071	*
5072	* @remarks No-long-jump zone!!!
5073	*/
5074	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5075	{
5076	/* Guest CR0. Guest FPU. */
5077	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5078
5079	/* Guest CR4. */
5080	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5081	AssertRCReturn(rc, rc);
5082
5083	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5084	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5085	{
5086	PVM pVM = pVCpu->CTX_SUFF(pVM);
5087	if ( pVM->hm.s.fNestedPaging
5088	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5089	{
5090	RTGCUINTREG uVal = 0;
5091	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5092	if (pMixedCtx->cr3 != uVal)
5093	{
5094	CPUMSetGuestCR3(pVCpu, uVal);
5095	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5096	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5097	}
5098
5099	/* We require EFER to check PAE mode. */
5100	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5101
5102	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5103	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5104	{
5105	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5106	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5107	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5108	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5109	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5110	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5111	}
5112	AssertRCReturn(rc, rc);
5113	}
5114
5115	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5116	}
5117	return rc;
5118	}
5119
5120
5121	/**
5122	* Reads a guest segment register from the current VMCS into the guest-CPU
5123	* context.
5124	*
5125	* @returns VBox status code.
5126	* @param pVCpu Pointer to the VMCPU.
5127	* @param idxSel Index of the selector in the VMCS.
5128	* @param idxLimit Index of the segment limit in the VMCS.
5129	* @param idxBase Index of the segment base in the VMCS.
5130	* @param idxAccess Index of the access rights of the segment in the VMCS.
5131	* @param pSelReg Pointer to the segment selector.
5132	*
5133	* @remarks No-long-jump zone!!!
5134	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5135	* as that takes care of whether to read from the VMCS cache or not.
5136	*/
5137	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5138	PCPUMSELREG pSelReg)
5139	{
5140	uint32_t u32Val = 0;
5141	int rc = VMXReadVmcs32(idxSel, &u32Val);
5142	pSelReg->Sel = (uint16_t)u32Val;
5143	pSelReg->ValidSel = (uint16_t)u32Val;
5144	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5145
5146	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5147	pSelReg->u32Limit = u32Val;
5148
5149	RTGCUINTREG uGCVal = 0;
5150	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5151	pSelReg->u64Base = uGCVal;
5152
5153	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5154	pSelReg->Attr.u = u32Val;
5155	AssertRCReturn(rc, rc);
5156
5157	/*
5158	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5159	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5160	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5161	*
5162	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5163	*/
5164	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5165	{
5166	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5167	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5168	}
5169	return rc;
5170	}
5171
5172
5173	/**
5174	* Saves the guest segment registers from the current VMCS into the guest-CPU
5175	* context.
5176	*
5177	* @returns VBox status code.
5178	* @param pVCpu Pointer to the VMCPU.
5179	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5180	* out-of-sync. Make sure to update the required fields
5181	* before using them.
5182	*
5183	* @remarks No-long-jump zone!!!
5184	*/
5185	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5186	{
5187	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5188	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5189	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5190	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5191	#else
5192	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5193	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5194	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5195	#endif
5196
5197	int rc = VINF_SUCCESS;
5198
5199	/* Guest segment registers. */
5200	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5201	{
5202	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5203	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5204	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5205	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5206	rc \|= VMXLOCAL_READ_SEG(ES, es);
5207	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5208	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5209	AssertRCReturn(rc, rc);
5210
5211	/* Restore segment attributes for real-on-v86 mode hack. */
5212	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5213	{
5214	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5215	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5216	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5217	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5218	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5219	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5220	}
5221	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5222	}
5223
5224	return rc;
5225	}
5226
5227
5228	/**
5229	* Saves the guest descriptor table registers and task register from the current
5230	* VMCS into the guest-CPU context.
5231	*
5232	* @returns VBox status code.
5233	* @param pVCpu Pointer to the VMCPU.
5234	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5235	* out-of-sync. Make sure to update the required fields
5236	* before using them.
5237	*
5238	* @remarks No-long-jump zone!!!
5239	*/
5240	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5241	{
5242	int rc = VINF_SUCCESS;
5243
5244	/* Guest LDTR. */
5245	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5246	{
5247	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5248	AssertRCReturn(rc, rc);
5249	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5250	}
5251
5252	/* Guest GDTR. */
5253	RTGCUINTREG uGCVal = 0;
5254	uint32_t u32Val = 0;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5256	{
5257	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5258	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5259	pMixedCtx->gdtr.pGdt = uGCVal;
5260	pMixedCtx->gdtr.cbGdt = u32Val;
5261	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5262	}
5263
5264	/* Guest IDTR. */
5265	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5266	{
5267	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5268	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5269	pMixedCtx->idtr.pIdt = uGCVal;
5270	pMixedCtx->idtr.cbIdt = u32Val;
5271	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5272	}
5273
5274	/* Guest TR. */
5275	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5276	{
5277	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5278
5279	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5280	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5281	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5282	AssertRCReturn(rc, rc);
5283	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5284	}
5285	return rc;
5286	}
5287
5288
5289	/**
5290	* Saves the guest debug registers from the current VMCS into the guest-CPU
5291	* context.
5292	*
5293	* @returns VBox status code.
5294	* @param pVCpu Pointer to the VMCPU.
5295	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5296	* out-of-sync. Make sure to update the required fields
5297	* before using them.
5298	*
5299	* @remarks No-long-jump zone!!!
5300	*/
5301	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5302	{
5303	int rc = VINF_SUCCESS;
5304	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5305	{
5306	RTGCUINTREG uVal;
5307	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5308	pMixedCtx->dr[7] = uVal;
5309
5310	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5311	}
5312	return rc;
5313	}
5314
5315
5316	/**
5317	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5318	*
5319	* @returns VBox status code.
5320	* @param pVCpu Pointer to the VMCPU.
5321	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5322	* out-of-sync. Make sure to update the required fields
5323	* before using them.
5324	*
5325	* @remarks No-long-jump zone!!!
5326	*/
5327	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5328	{
5329	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5330	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5331	return VINF_SUCCESS;
5332	}
5333
5334
5335	/**
5336	* Saves the entire guest state from the currently active VMCS into the
5337	* guest-CPU context. This essentially VMREADs all guest-data.
5338	*
5339	* @returns VBox status code.
5340	* @param pVCpu Pointer to the VMCPU.
5341	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5342	* out-of-sync. Make sure to update the required fields
5343	* before using them.
5344	*/
5345	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5346	{
5347	Assert(pVCpu);
5348	Assert(pMixedCtx);
5349
5350	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5351	return VINF_SUCCESS;
5352
5353	VMMRZCallRing3Disable(pVCpu);
5354	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5355	LogFunc(("\n"));
5356
5357	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5358	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5359
5360	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5361	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5362
5363	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5364	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5365
5366	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5367	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5368
5369	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5370	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5371
5372	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5373	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5374
5375	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5376	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5377
5378	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5379	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5380
5381	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5383
5384	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5385	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5386
5387	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5388	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5389
5390	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5391	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5392
5393	VMMRZCallRing3Enable(pVCpu);
5394	return rc;
5395	}
5396
5397
5398	/**
5399	* Check per-VM and per-VCPU force flag actions that require us to go back to
5400	* ring-3 for one reason or another.
5401	*
5402	* @returns VBox status code (information status code included).
5403	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5404	* ring-3.
5405	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5406	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5407	* interrupts)
5408	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5409	* all EMTs to be in ring-3.
5410	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5411	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5412	* to the EM loop.
5413	*
5414	* @param pVM Pointer to the VM.
5415	* @param pVCpu Pointer to the VMCPU.
5416	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5417	* out-of-sync. Make sure to update the required fields
5418	* before using them.
5419	*/
5420	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5421	{
5422	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5423
5424	int rc = VERR_INTERNAL_ERROR_5;
5425	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5426	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5427	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5428	{
5429	/* We need the control registers now, make sure the guest-CPU context is updated. */
5430	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5431	AssertRCReturn(rc, rc);
5432
5433	/* Pending HM CR3 sync. */
5434	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5435	{
5436	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5437	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5438	}
5439	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5440	{
5441	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5442	AssertRC(rc);
5443	}
5444
5445	/* Pending PGM C3 sync. */
5446	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5447	{
5448	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5449	if (rc != VINF_SUCCESS)
5450	{
5451	AssertRC(rc);
5452	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5453	return rc;
5454	}
5455	}
5456
5457	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5458	/* -XXX- what was that about single stepping? */
5459	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5460	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5461	{
5462	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5463	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5464	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5465	return rc;
5466	}
5467
5468	/* Pending VM request packets, such as hardware interrupts. */
5469	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5470	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5471	{
5472	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5473	return VINF_EM_PENDING_REQUEST;
5474	}
5475
5476	/* Pending PGM pool flushes. */
5477	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5478	{
5479	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5480	return VINF_PGM_POOL_FLUSH_PENDING;
5481	}
5482
5483	/* Pending DMA requests. */
5484	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5485	{
5486	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5487	return VINF_EM_RAW_TO_R3;
5488	}
5489	}
5490
5491	/* Paranoia. */
5492	Assert(rc != VERR_EM_INTERPRETER);
5493	return VINF_SUCCESS;
5494	}
5495
5496
5497	/**
5498	* Converts any TRPM trap into a pending VMX event. This is typically used when
5499	* entering from ring-3 (not longjmp returns).
5500	*
5501	* @param pVCpu Pointer to the VMCPU.
5502	*/
5503	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5504	{
5505	Assert(TRPMHasTrap(pVCpu));
5506	Assert(!pVCpu->hm.s.Event.fPending);
5507
5508	uint8_t uVector;
5509	TRPMEVENT enmTrpmEvent;
5510	RTGCUINT uErrCode;
5511	RTGCUINTPTR GCPtrFaultAddress;
5512	uint8_t cbInstr;
5513
5514	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5515	AssertRC(rc);
5516
5517	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5518	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5519	if (enmTrpmEvent == TRPM_TRAP)
5520	{
5521	switch (uVector)
5522	{
5523	case X86_XCPT_BP:
5524	case X86_XCPT_OF:
5525	{
5526	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5527	break;
5528	}
5529
5530	case X86_XCPT_PF:
5531	case X86_XCPT_DF:
5532	case X86_XCPT_TS:
5533	case X86_XCPT_NP:
5534	case X86_XCPT_SS:
5535	case X86_XCPT_GP:
5536	case X86_XCPT_AC:
5537	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5538	/* no break! */
5539	default:
5540	{
5541	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5542	break;
5543	}
5544	}
5545	}
5546	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5547	{
5548	if (uVector != X86_XCPT_NMI)
5549	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5550	else
5551	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5552	}
5553	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5554	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5555	else
5556	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5557
5558	rc = TRPMResetTrap(pVCpu);
5559	AssertRC(rc);
5560	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5561	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5562	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5563	}
5564
5565
5566	/**
5567	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5568	* VT-x to execute any instruction.
5569	*
5570	* @param pvCpu Pointer to the VMCPU.
5571	*/
5572	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5573	{
5574	Assert(pVCpu->hm.s.Event.fPending);
5575
5576	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5577	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5578	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5579	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5580
5581	/* If a trap was already pending, we did something wrong! */
5582	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5583
5584	TRPMEVENT enmTrapType;
5585	switch (uVectorType)
5586	{
5587	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5588	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5589	enmTrapType = TRPM_HARDWARE_INT;
5590	break;
5591	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5592	enmTrapType = TRPM_SOFTWARE_INT;
5593	break;
5594	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5595	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5596	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5597	enmTrapType = TRPM_TRAP;
5598	break;
5599	default:
5600	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5601	enmTrapType = TRPM_32BIT_HACK;
5602	break;
5603	}
5604
5605	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5606	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5607	AssertRC(rc);
5608
5609	if (fErrorCodeValid)
5610	TRPMSetErrorCode(pVCpu, uErrorCode);
5611	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5612	&& uVector == X86_XCPT_PF)
5613	{
5614	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5615	}
5616	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5617	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5618	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5619	{
5620	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5621	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5622	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5623	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5624	}
5625	pVCpu->hm.s.Event.fPending = false;
5626	}
5627
5628
5629	/**
5630	* Does the necessary state syncing before doing a longjmp to ring-3.
5631	*
5632	* @param pVM Pointer to the VM.
5633	* @param pVCpu Pointer to the VMCPU.
5634	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5635	* out-of-sync. Make sure to update the required fields
5636	* before using them.
5637	* @param rcExit The reason for exiting to ring-3. Can be
5638	* VINF_VMM_UNKNOWN_RING3_CALL.
5639	*
5640	* @remarks No-long-jmp zone!!!
5641	*/
5642	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5643	{
5644	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5645	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5646
5647	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5648	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5649	AssertRC(rc);
5650
5651	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5652	if (CPUMIsGuestFPUStateActive(pVCpu))
5653	{
5654	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5655	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5656	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5657	}
5658
5659	/* Restore debug registers if necessary and resync on next R0 reentry. */
5660	if (CPUMIsGuestDebugStateActive(pVCpu))
5661	{
5662	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5663	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5664	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5665	}
5666	else if (CPUMIsHyperDebugStateActive(pVCpu))
5667	{
5668	CPUMR0LoadHostDebugState(pVM, pVCpu);
5669	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5670	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5671	}
5672
5673	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5674	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5675	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5676	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5677	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5678	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5679	}
5680
5681
5682	/**
5683	* An action requires us to go back to ring-3. This function does the necessary
5684	* steps before we can safely return to ring-3. This is not the same as longjmps
5685	* to ring-3, this is voluntary.
5686	*
5687	* @param pVM Pointer to the VM.
5688	* @param pVCpu Pointer to the VMCPU.
5689	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5690	* out-of-sync. Make sure to update the required fields
5691	* before using them.
5692	* @param rcExit The reason for exiting to ring-3. Can be
5693	* VINF_VMM_UNKNOWN_RING3_CALL.
5694	*/
5695	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5696	{
5697	Assert(pVM);
5698	Assert(pVCpu);
5699	Assert(pMixedCtx);
5700	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5701
5702	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5703	{
5704	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5705	return;
5706	}
5707	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5708	{
5709	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5710	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5711	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5712	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5713	return;
5714	}
5715
5716	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5717	VMMRZCallRing3Disable(pVCpu);
5718	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5719
5720	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5721	if (pVCpu->hm.s.Event.fPending)
5722	{
5723	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5724	Assert(!pVCpu->hm.s.Event.fPending);
5725	}
5726
5727	/* Sync. the guest state. */
5728	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5729	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5730
5731	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5732	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5733	\| CPUM_CHANGED_LDTR
5734	\| CPUM_CHANGED_GDTR
5735	\| CPUM_CHANGED_IDTR
5736	\| CPUM_CHANGED_TR
5737	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5738
5739	/* On our way back from ring-3 the following needs to be done. */
5740	/** @todo This can change with preemption hooks. */
5741	if (rcExit == VINF_EM_RAW_INTERRUPT)
5742	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5743	else
5744	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5745
5746	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5747	VMMRZCallRing3Enable(pVCpu);
5748	}
5749
5750
5751	/**
5752	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5753	* longjump to ring-3 and possibly get preempted.
5754	*
5755	* @param pVCpu Pointer to the VMCPU.
5756	* @param enmOperation The operation causing the ring-3 longjump.
5757	* @param pvUser The user argument (pointer to the possibly
5758	* out-of-date guest-CPU context).
5759	*
5760	* @remarks Must never be called with @a enmOperation ==
5761	* VMMCALLRING3_VM_R0_ASSERTION.
5762	*/
5763	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5764	{
5765	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5766	Assert(pVCpu);
5767	Assert(pvUser);
5768	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5769	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5770
5771	VMMRZCallRing3Disable(pVCpu);
5772	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5773	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5774	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5775	VMMRZCallRing3Enable(pVCpu);
5776	}
5777
5778
5779	/**
5780	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5781	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5782	*
5783	* @returns VBox status code.
5784	* @param pVCpu Pointer to the VMCPU.
5785	*/
5786	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5787	{
5788	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5789	{
5790	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5791	{
5792	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5793	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5794	AssertRC(rc);
5795	}
5796	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5797	}
5798
5799
5800	/**
5801	* Injects any pending events into the guest if the guest is in a state to
5802	* receive them.
5803	*
5804	* @returns VBox status code (informational status codes included).
5805	* @param pVCpu Pointer to the VMCPU.
5806	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5807	* out-of-sync. Make sure to update the required fields
5808	* before using them.
5809	*/
5810	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5811	{
5812	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5813	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5814	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5815	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5816
5817	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5818	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5819	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5820	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5821	Assert(!TRPMHasTrap(pVCpu));
5822
5823	int rc = VINF_SUCCESS;
5824	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5825	{
5826	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5827	bool fInject = true;
5828	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5829	{
5830	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5831	AssertRCReturn(rc, rc);
5832	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5833	if ( fBlockInt
5834	\|\| fBlockSti
5835	\|\| fBlockMovSS)
5836	{
5837	fInject = false;
5838	}
5839	}
5840	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5841	&& ( fBlockMovSS
5842	\|\| fBlockSti))
5843	{
5844	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5845	fInject = false;
5846	}
5847
5848	if (fInject)
5849	{
5850	Log(("Injecting pending event\n"));
5851	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5852	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5853	AssertRCReturn(rc, rc);
5854	pVCpu->hm.s.Event.fPending = false;
5855	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5856	}
5857	else
5858	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5859	} /** @todo SMI. SMIs take priority over NMIs. */
5860	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5861	{
5862	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5863	if ( !fBlockMovSS
5864	&& !fBlockSti)
5865	{
5866	Log(("Injecting NMI\n"));
5867	RTGCUINTPTR uIntrInfo;
5868	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5869	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5870	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5871	0 /* GCPtrFaultAddress */, &uIntrState);
5872	AssertRCReturn(rc, rc);
5873	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5874	}
5875	else
5876	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5877	}
5878	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5879	{
5880	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5881	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5882	AssertRCReturn(rc, rc);
5883	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5884	if ( !fBlockInt
5885	&& !fBlockSti
5886	&& !fBlockMovSS)
5887	{
5888	uint8_t u8Interrupt;
5889	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5890	if (RT_SUCCESS(rc))
5891	{
5892	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5893	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5894	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5895	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5896	0 /* GCPtrFaultAddress */, &uIntrState);
5897	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5898	}
5899	else
5900	{
5901	/** @todo Does this actually happen? If not turn it into an assertion. */
5902	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5903	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5904	rc = VINF_SUCCESS;
5905	}
5906	}
5907	else
5908	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5909	}
5910
5911	/*
5912	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5913	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5914	*/
5915	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5916	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5917	int rc2 = VINF_SUCCESS;
5918	if ( fBlockSti
5919	\|\| fBlockMovSS)
5920	{
5921	if (!DBGFIsStepping(pVCpu))
5922	{
5923	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5924	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5925	{
5926	/*
5927	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5928	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5929	*/
5930	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5931	}
5932	}
5933	else
5934	{
5935	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5936	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5937	uIntrState = 0;
5938	}
5939	}
5940
5941	/*
5942	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5943	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5944	*/
5945	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5946	AssertRC(rc2);
5947
5948	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5949	return rc;
5950	}
5951
5952
5953	/**
5954	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5955	*
5956	* @param pVCpu Pointer to the VMCPU.
5957	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5958	* out-of-sync. Make sure to update the required fields
5959	* before using them.
5960	*/
5961	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5962	{
5963	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5964	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5965	}
5966
5967
5968	/**
5969	* Injects a double-fault (#DF) exception into the VM.
5970	*
5971	* @returns VBox status code (informational status code included).
5972	* @param pVCpu Pointer to the VMCPU.
5973	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5974	* out-of-sync. Make sure to update the required fields
5975	* before using them.
5976	*/
5977	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5978	{
5979	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5980	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5981	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5982	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5983	puIntrState);
5984	}
5985
5986
5987	/**
5988	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5989	*
5990	* @param pVCpu Pointer to the VMCPU.
5991	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5992	* out-of-sync. Make sure to update the required fields
5993	* before using them.
5994	*/
5995	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5996	{
5997	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5998	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5999	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6000	}
6001
6002
6003	/**
6004	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6005	*
6006	* @param pVCpu Pointer to the VMCPU.
6007	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6008	* out-of-sync. Make sure to update the required fields
6009	* before using them.
6010	* @param cbInstr The value of RIP that is to be pushed on the guest
6011	* stack.
6012	*/
6013	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6014	{
6015	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6016	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6017	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6018	}
6019
6020
6021	/**
6022	* Injects a general-protection (#GP) fault into the VM.
6023	*
6024	* @returns VBox status code (informational status code included).
6025	* @param pVCpu Pointer to the VMCPU.
6026	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6027	* out-of-sync. Make sure to update the required fields
6028	* before using them.
6029	* @param u32ErrorCode The error code associated with the #GP.
6030	*/
6031	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6032	uint32_t *puIntrState)
6033	{
6034	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6035	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6036	if (fErrorCodeValid)
6037	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6038	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6039	puIntrState);
6040	}
6041
6042
6043	/**
6044	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6045	*
6046	* @param pVCpu Pointer to the VMCPU.
6047	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6048	* out-of-sync. Make sure to update the required fields
6049	* before using them.
6050	* @param uVector The software interrupt vector number.
6051	* @param cbInstr The value of RIP that is to be pushed on the guest
6052	* stack.
6053	*/
6054	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6055	{
6056	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6057	if ( uVector == X86_XCPT_BP
6058	\|\| uVector == X86_XCPT_OF)
6059	{
6060	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6061	}
6062	else
6063	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6064	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6065	}
6066
6067
6068	/**
6069	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6070	* stack.
6071	*
6072	* @returns VBox status code (information status code included).
6073	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6074	* @param pVM Pointer to the VM.
6075	* @param pMixedCtx Pointer to the guest-CPU context.
6076	* @param uValue The value to push to the guest stack.
6077	*/
6078	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6079	{
6080	/*
6081	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6082	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6083	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6084	*/
6085	if (pMixedCtx->sp == 1)
6086	return VINF_EM_RESET;
6087	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6088	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6089	AssertRCReturn(rc, rc);
6090	return rc;
6091	}
6092
6093
6094	/**
6095	* Injects an event into the guest upon VM-entry by updating the relevant fields
6096	* in the VM-entry area in the VMCS.
6097	*
6098	* @returns VBox status code (informational error codes included).
6099	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6100	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6101	*
6102	* @param pVCpu Pointer to the VMCPU.
6103	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6104	* be out-of-sync. Make sure to update the required
6105	* fields before using them.
6106	* @param u64IntrInfo The VM-entry interruption-information field.
6107	* @param cbInstr The VM-entry instruction length in bytes (for
6108	* software interrupts, exceptions and privileged
6109	* software exceptions).
6110	* @param u32ErrCode The VM-entry exception error code.
6111	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6112	* @param puIntrState Pointer to the current guest interruptibility-state.
6113	* This interruptibility-state will be updated if
6114	* necessary. This cannot not be NULL.
6115	*
6116	* @remarks No-long-jump zone!!!
6117	* @remarks Requires CR0!
6118	*/
6119	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6120	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6121	{
6122	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6123	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6124	Assert(puIntrState);
6125	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6126
6127	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6128	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6129
6130	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6131	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6132	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6133
6134	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6135
6136	/* We require CR0 to check if the guest is in real-mode. */
6137	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6138	AssertRCReturn(rc, rc);
6139
6140	/*
6141	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6142	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6143	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6144	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6145	*/
6146	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6147	{
6148	PVM pVM = pVCpu->CTX_SUFF(pVM);
6149	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6150	{
6151	Assert(PDMVmmDevHeapIsEnabled(pVM));
6152	Assert(pVM->hm.s.vmx.pRealModeTSS);
6153
6154	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6155	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6156	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6157	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6158	AssertRCReturn(rc, rc);
6159	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6160
6161	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6162	const size_t cbIdtEntry = 4;
6163	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6164	{
6165	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6166	if (uVector == X86_XCPT_DF)
6167	return VINF_EM_RESET;
6168	else if (uVector == X86_XCPT_GP)
6169	{
6170	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6171	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6172	}
6173
6174	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6175	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6176	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6177	}
6178
6179	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6180	uint16_t uGuestIp = pMixedCtx->ip;
6181	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6182	{
6183	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6184	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6185	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6186	}
6187	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6188	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6189
6190	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6191	uint16_t offIdtEntry = 0;
6192	RTSEL selIdtEntry = 0;
6193	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6194	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6195	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6196	AssertRCReturn(rc, rc);
6197
6198	/* Construct the stack frame for the interrupt/exception handler. */
6199	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6200	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6201	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6202	AssertRCReturn(rc, rc);
6203
6204	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6205	if (rc == VINF_SUCCESS)
6206	{
6207	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6208	pMixedCtx->rip = offIdtEntry;
6209	pMixedCtx->cs.Sel = selIdtEntry;
6210	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6211	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6212	&& uVector == X86_XCPT_PF)
6213	{
6214	pMixedCtx->cr2 = GCPtrFaultAddress;
6215	}
6216	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6217	\| HM_CHANGED_GUEST_RIP
6218	\| HM_CHANGED_GUEST_RFLAGS
6219	\| HM_CHANGED_GUEST_RSP;
6220
6221	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6222	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6223	{
6224	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6225	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6226	Log(("Clearing inhibition due to STI.\n"));
6227	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6228	}
6229	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6230	}
6231	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6232	return rc;
6233	}
6234	else
6235	{
6236	/*
6237	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6238	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6239	*/
6240	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6241	}
6242	}
6243
6244	/* Validate. */
6245	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6246	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6247	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6248
6249	/* Inject. */
6250	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6251	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6252	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6253	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6254
6255	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6256	&& uVector == X86_XCPT_PF)
6257	{
6258	pMixedCtx->cr2 = GCPtrFaultAddress;
6259	}
6260	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6261
6262	AssertRCReturn(rc, rc);
6263	return rc;
6264	}
6265
6266
6267	/**
6268	* Enters the VT-x session.
6269	*
6270	* @returns VBox status code.
6271	* @param pVM Pointer to the VM.
6272	* @param pVCpu Pointer to the VMCPU.
6273	* @param pCpu Pointer to the CPU info struct.
6274	*/
6275	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6276	{
6277	AssertPtr(pVM);
6278	AssertPtr(pVCpu);
6279	Assert(pVM->hm.s.vmx.fSupported);
6280	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6281	NOREF(pCpu);
6282
6283	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6284
6285	/* Make sure we're in VMX root mode. */
6286	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6287	if (!(u32HostCR4 & X86_CR4_VMXE))
6288	{
6289	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6290	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6291	}
6292
6293	/* Load the active VMCS as the current one. */
6294	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6295	if (RT_FAILURE(rc))
6296	return rc;
6297
6298	/** @todo this will change with preemption hooks where can can VMRESUME as long
6299	* as we're no preempted. */
6300	pVCpu->hm.s.fResumeVM = false;
6301	return VINF_SUCCESS;
6302	}
6303
6304
6305	/**
6306	* Leaves the VT-x session.
6307	*
6308	* @returns VBox status code.
6309	* @param pVM Pointer to the VM.
6310	* @param pVCpu Pointer to the VMCPU.
6311	* @param pCtx Pointer to the guest-CPU context.
6312	*/
6313	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6314	{
6315	AssertPtr(pVCpu);
6316	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6317	NOREF(pVM);
6318	NOREF(pCtx);
6319
6320	/** @todo this will change with preemption hooks where we only VMCLEAR when
6321	* we are actually going to be preempted, not all the time like we
6322	* currently do. */
6323	/*
6324	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6325	* and mark the VMCS launch-state as "clear".
6326	*/
6327	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6328	return rc;
6329	}
6330
6331
6332	/**
6333	* Saves the host state in the VMCS host-state.
6334	* Sets up the VM-exit MSR-load area.
6335	*
6336	* The CPU state will be loaded from these fields on every successful VM-exit.
6337	*
6338	* @returns VBox status code.
6339	* @param pVM Pointer to the VM.
6340	* @param pVCpu Pointer to the VMCPU.
6341	*
6342	* @remarks No-long-jump zone!!!
6343	*/
6344	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6345	{
6346	AssertPtr(pVM);
6347	AssertPtr(pVCpu);
6348	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6349
6350	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6351
6352	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6353	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6354	return VINF_SUCCESS;
6355
6356	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6357	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6358
6359	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6360	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6361
6362	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6363	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6364
6365	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6366	return rc;
6367	}
6368
6369
6370	/**
6371	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6372	* loaded from these fields on every successful VM-entry.
6373	*
6374	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6375	* Sets up the VM-entry controls.
6376	* Sets up the appropriate VMX non-root function to execute guest code based on
6377	* the guest CPU mode.
6378	*
6379	* @returns VBox status code.
6380	* @param pVM Pointer to the VM.
6381	* @param pVCpu Pointer to the VMCPU.
6382	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6383	* out-of-sync. Make sure to update the required fields
6384	* before using them.
6385	*
6386	* @remarks No-long-jump zone!!!
6387	*/
6388	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6389	{
6390	AssertPtr(pVM);
6391	AssertPtr(pVCpu);
6392	AssertPtr(pMixedCtx);
6393	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6394
6395	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6396
6397	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6398
6399	/* Determine real-on-v86 mode. */
6400	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6401	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6402	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6403	{
6404	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6405	}
6406
6407	/*
6408	* Load the guest-state into the VMCS.
6409	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6410	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6411	*/
6412	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6413	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6414
6415	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6416	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6417
6418	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6419	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6420
6421	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6422	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6423
6424	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6425	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6427
6428	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6429	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6430
6431	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6432	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6433
6434	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6435	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6436
6437	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6438	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6439
6440	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6441	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6442
6443	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6444	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6445	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6446
6447	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6448	return rc;
6449	}
6450
6451
6452	/**
6453	* Does the preparations before executing guest code in VT-x.
6454	*
6455	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6456	* recompiler. We must be cautious what we do here regarding committing
6457	* guest-state information into the the VMCS assuming we assuredly execute the
6458	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6459	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6460	* that the recompiler can (and should) use them when it resumes guest
6461	* execution. Otherwise such operations must be done when we can no longer
6462	* exit to ring-3.
6463	*
6464	* @returns VBox status code (informational status codes included).
6465	* @retval VINF_SUCCESS if we can proceed with running the guest.
6466	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6467	* into the guest.
6468	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6469	*
6470	* @param pVM Pointer to the VM.
6471	* @param pVCpu Pointer to the VMCPU.
6472	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6473	* out-of-sync. Make sure to update the required fields
6474	* before using them.
6475	* @param pVmxTransient Pointer to the VMX transient structure.
6476	*
6477	* @remarks Called with preemption disabled.
6478	*/
6479	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6480	{
6481	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6482
6483	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6484	PGMRZDynMapFlushAutoSet(pVCpu);
6485	#endif
6486
6487	/* Check force flag actions that might require us to go back to ring-3. */
6488	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6489	if (rc != VINF_SUCCESS)
6490	return rc;
6491
6492	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6493	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6494	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6495	{
6496	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6497	RTGCPHYS GCPhysApicBase;
6498	GCPhysApicBase = pMixedCtx->msrApicBase;
6499	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6500
6501	/* Unalias any existing mapping. */
6502	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6503	AssertRCReturn(rc, rc);
6504
6505	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6506	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6507	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6508	AssertRCReturn(rc, rc);
6509
6510	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6511	}
6512
6513	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6514	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6515	pVmxTransient->uEFlags = ASMIntDisableFlags();
6516	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6517	{
6518	ASMSetFlags(pVmxTransient->uEFlags);
6519	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6520	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6521	return VINF_EM_RAW_INTERRUPT;
6522	}
6523	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6524	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6525	#endif
6526
6527	/*
6528	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6529	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6530	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6531	*/
6532	/** @todo Rework event evaluation and injection to be complete separate. */
6533	if (TRPMHasTrap(pVCpu))
6534	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6535
6536	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6537	AssertRCReturn(rc, rc);
6538	return rc;
6539	}
6540
6541
6542	/**
6543	* Prepares to run guest code in VT-x and we've committed to doing so. This
6544	* means there is no backing out to ring-3 or anywhere else at this
6545	* point.
6546	*
6547	* @param pVM Pointer to the VM.
6548	* @param pVCpu Pointer to the VMCPU.
6549	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6550	* out-of-sync. Make sure to update the required fields
6551	* before using them.
6552	* @param pVmxTransient Pointer to the VMX transient structure.
6553	*
6554	* @remarks Called with preemption disabled.
6555	* @remarks No-long-jump zone!!!
6556	*/
6557	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6558	{
6559	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6560	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6561
6562	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6563	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6564	pVmxTransient->uEFlags = ASMIntDisableFlags();
6565	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6566	#endif
6567
6568	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6569	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6570	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6571	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6572	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6573	#endif
6574	int rc = VINF_SUCCESS;
6575	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6576	{
6577	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6578	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6579	}
6580	else if (pVCpu->hm.s.fContextUseFlags)
6581	{
6582	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6583	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6584	}
6585	AssertRC(rc);
6586	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6587
6588	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6590	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6591
6592	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6593	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6594	{
6595	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6596	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6597	}
6598
6599	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6600	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6601	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6602
6603	/*
6604	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6605	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6606	*/
6607	if (pVM->hm.s.fTPRPatchingActive)
6608	{
6609	Assert(!CPUMIsGuestInLongMode(pVCpu));
6610
6611	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6612	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6613	AssertRC(rc);
6614
6615	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6616	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6617	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6618	see hmR0VmxLoadGuestApicState(). */
6619	}
6620
6621	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6622	/*
6623	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6624	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6625	*/
6626	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6627	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6628	{
6629	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6630	uint64_t u64HostTscAux = 0;
6631	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6632	AssertRC(rc2);
6633	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6634	}
6635	#endif
6636
6637	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6638	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6639	to start executing. */
6640	}
6641
6642
6643	/**
6644	* Performs some essential restoration of state after running guest code in
6645	* VT-x.
6646	*
6647	* @param pVM Pointer to the VM.
6648	* @param pVCpu Pointer to the VMCPU.
6649	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6650	* out-of-sync. Make sure to update the required fields
6651	* before using them.
6652	* @param pVmxTransient Pointer to the VMX transient structure.
6653	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6654	*
6655	* @remarks Called with interrupts disabled.
6656	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6657	* unconditionally when it is safe to do so.
6658	*/
6659	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6660	{
6661	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6662	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6663
6664	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6665	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6666	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6667	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6668	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6669
6670	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6671	{
6672	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6673	/* Restore host's TSC_AUX. */
6674	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6675	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6676	#endif
6677	/** @todo Find a way to fix hardcoding a guestimate. */
6678	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6679	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6680	}
6681
6682	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6683	Assert(!(ASMGetFlags() & X86_EFL_IF));
6684	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6685
6686	/* Restore the effects of TPR patching if any. */
6687	if (pVM->hm.s.fTPRPatchingActive)
6688	{
6689	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6690	AssertRC(rc);
6691	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6692	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6693	}
6694
6695	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6696	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6697
6698	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6699	uint32_t uExitReason;
6700	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6701	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6702	AssertRC(rc);
6703	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6704	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6705
6706	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6707	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6708
6709	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6710	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6711	{
6712	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6713	return;
6714	}
6715
6716	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6717	{
6718	/* Update the guest interruptibility-state from the VMCS. */
6719	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6720	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6721	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6722	AssertRC(rc);
6723	#endif
6724	/*
6725	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6726	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6727	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6728	*/
6729	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6730	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6731	{
6732	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6733	AssertRC(rc);
6734	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6735	}
6736	}
6737	}
6738
6739
6740	/**
6741	* Runs the guest code using VT-x.
6742	*
6743	* @returns VBox status code.
6744	* @param pVM Pointer to the VM.
6745	* @param pVCpu Pointer to the VMCPU.
6746	* @param pCtx Pointer to the guest-CPU context.
6747	*
6748	* @remarks Called with preemption disabled.
6749	*/
6750	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6751	{
6752	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6753	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6754
6755	VMXTRANSIENT VmxTransient;
6756	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6757	int rc = VERR_INTERNAL_ERROR_5;
6758	uint32_t cLoops = 0;
6759
6760	for (;; cLoops++)
6761	{
6762	Assert(!HMR0SuspendPending());
6763	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6764	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6765	(unsigned)RTMpCpuId(), cLoops));
6766
6767	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6768	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6769	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6770	if (rc != VINF_SUCCESS)
6771	break;
6772
6773	/*
6774	* No longjmps to ring-3 from this point on!!!
6775	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6776	* This also disables flushing of the R0-logger instance (if any).
6777	*/
6778	VMMRZCallRing3Disable(pVCpu);
6779	VMMRZCallRing3RemoveNotification(pVCpu);
6780	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6781
6782	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6783	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6784
6785	/*
6786	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6787	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6788	*/
6789	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6790	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6791	{
6792	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6793	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6794	return rc;
6795	}
6796
6797	/* Handle the VM-exit. */
6798	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6799	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6800	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6801	HMVMX_START_EXIT_DISPATCH_PROF();
6802	#ifdef HMVMX_USE_FUNCTION_TABLE
6803	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6804	#else
6805	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6806	#endif
6807	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6808	if (rc != VINF_SUCCESS)
6809	break;
6810	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6811	{
6812	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6813	rc = VINF_EM_RAW_INTERRUPT;
6814	break;
6815	}
6816	}
6817
6818	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6819	if (rc == VERR_EM_INTERPRETER)
6820	rc = VINF_EM_RAW_EMULATE_INSTR;
6821	else if (rc == VINF_EM_RESET)
6822	rc = VINF_EM_TRIPLE_FAULT;
6823	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6824	return rc;
6825	}
6826
6827
6828	#ifndef HMVMX_USE_FUNCTION_TABLE
6829	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6830	{
6831	int rc;
6832	switch (rcReason)
6833	{
6834	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6842	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6853	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6854	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6856	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6857	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6858	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6859	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6860	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6861	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6862	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6863	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6864	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6865	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6866	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6867
6868	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6869	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6870	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6871	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6872	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6873	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6874	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6875	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6876	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6877
6878	case VMX_EXIT_VMCALL:
6879	case VMX_EXIT_VMCLEAR:
6880	case VMX_EXIT_VMLAUNCH:
6881	case VMX_EXIT_VMPTRLD:
6882	case VMX_EXIT_VMPTRST:
6883	case VMX_EXIT_VMREAD:
6884	case VMX_EXIT_VMRESUME:
6885	case VMX_EXIT_VMWRITE:
6886	case VMX_EXIT_VMXOFF:
6887	case VMX_EXIT_VMXON:
6888	case VMX_EXIT_INVEPT:
6889	case VMX_EXIT_INVVPID:
6890	case VMX_EXIT_VMFUNC:
6891	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6892	break;
6893	default:
6894	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6895	break;
6896	}
6897	return rc;
6898	}
6899	#endif
6900
6901	#ifdef DEBUG
6902	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6903	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6904	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6905	# define VMX_ASSERT_PREEMPT_CPUID() \
6906	do \
6907	{ \
6908	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6909	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6910	} while (0)
6911
6912	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6913	do { \
6914	AssertPtr(pVCpu); \
6915	AssertPtr(pMixedCtx); \
6916	AssertPtr(pVmxTransient); \
6917	Assert(pVmxTransient->fVMEntryFailed == false); \
6918	Assert(ASMIntAreEnabled()); \
6919	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6920	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6921	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6922	(unsigned)pVCpu->idCpu)); \
6923	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6924	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6925	VMX_ASSERT_PREEMPT_CPUID(); \
6926	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6927	} while (0)
6928	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6929	do { \
6930	LogFunc(("\n")); \
6931	} while(0)
6932	#else /* Release builds */
6933	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6934	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6935	#endif
6936
6937
6938	/**
6939	* Advances the guest RIP after reading it from the VMCS.
6940	*
6941	* @returns VBox status code.
6942	* @param pVCpu Pointer to the VMCPU.
6943	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6944	* out-of-sync. Make sure to update the required fields
6945	* before using them.
6946	* @param pVmxTransient Pointer to the VMX transient structure.
6947	*
6948	* @remarks No-long-jump zone!!!
6949	*/
6950	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6951	{
6952	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6953	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6954	AssertRCReturn(rc, rc);
6955
6956	pMixedCtx->rip += pVmxTransient->cbInstr;
6957	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6958	return rc;
6959	}
6960
6961
6962	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6963	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6964	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6965	/**
6966	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6967	*/
6968	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6969	{
6970	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6971	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6972	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6973	Assert(ASMIntAreEnabled());
6974	return VINF_SUCCESS;
6975	#else
6976	return VINF_EM_RAW_INTERRUPT;
6977	#endif
6978	}
6979
6980
6981	/**
6982	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6983	*/
6984	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6985	{
6986	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6987	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6988
6989	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6990	AssertRCReturn(rc, rc);
6991
6992	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6993	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6994	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6995
6996	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6997	{
6998	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6999	return VINF_EM_RAW_INTERRUPT;
7000	}
7001
7002	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7003	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7004	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7005	{
7006	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7007	return VINF_SUCCESS;
7008	}
7009	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7010	{
7011	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7012	return rc;
7013	}
7014
7015	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7016	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7017	switch (uIntrType)
7018	{
7019	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7020	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7021	/* no break */
7022	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7023	{
7024	switch (uVector)
7025	{
7026	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7027	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7028	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7029	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7030	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7031	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7032	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7033	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7034	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7035	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7036	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7037	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7038	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7039	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7040	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7041	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7042	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7043	#endif
7044	default:
7045	{
7046	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7047	AssertRCReturn(rc, rc);
7048
7049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7050	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7051	{
7052	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7053	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7054	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7055	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7056	AssertRCReturn(rc, rc);
7057	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7058	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7059	0 /* GCPtrFaultAddress */);
7060	AssertRCReturn(rc, rc);
7061	}
7062	else
7063	{
7064	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7065	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7066	}
7067	break;
7068	}
7069	}
7070	break;
7071	}
7072
7073	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7074	default:
7075	{
7076	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7077	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7078	break;
7079	}
7080	}
7081	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7082	return rc;
7083	}
7084
7085
7086	/**
7087	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7088	*/
7089	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7090	{
7091	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7092
7093	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7094	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7095	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7096	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7097	AssertRCReturn(rc, rc);
7098
7099	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7101	return VINF_SUCCESS;
7102	}
7103
7104
7105	/**
7106	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7107	*/
7108	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7109	{
7110	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7111	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7112	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7113	}
7114
7115
7116	/**
7117	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7118	*/
7119	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7120	{
7121	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7123	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7124	}
7125
7126
7127	/**
7128	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7129	*/
7130	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7131	{
7132	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7133	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7134	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7135	}
7136
7137
7138	/**
7139	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7140	*/
7141	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7142	{
7143	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7144	PVM pVM = pVCpu->CTX_SUFF(pVM);
7145	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7146	if (RT_LIKELY(rc == VINF_SUCCESS))
7147	{
7148	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7149	Assert(pVmxTransient->cbInstr == 2);
7150	}
7151	else
7152	{
7153	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7154	rc = VERR_EM_INTERPRETER;
7155	}
7156	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7157	return rc;
7158	}
7159
7160
7161	/**
7162	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7163	*/
7164	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7165	{
7166	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7167	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7168	AssertRCReturn(rc, rc);
7169
7170	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7171	return VINF_EM_RAW_EMULATE_INSTR;
7172
7173	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7174	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7175	}
7176
7177
7178	/**
7179	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7180	*/
7181	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7182	{
7183	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7184	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7185	AssertRCReturn(rc, rc);
7186
7187	PVM pVM = pVCpu->CTX_SUFF(pVM);
7188	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7189	if (RT_LIKELY(rc == VINF_SUCCESS))
7190	{
7191	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7192	Assert(pVmxTransient->cbInstr == 2);
7193	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7194	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7195	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7196	}
7197	else
7198	{
7199	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7200	rc = VERR_EM_INTERPRETER;
7201	}
7202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7203	return rc;
7204	}
7205
7206
7207	/**
7208	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7209	*/
7210	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7211	{
7212	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7213	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7214	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7215	AssertRCReturn(rc, rc);
7216
7217	PVM pVM = pVCpu->CTX_SUFF(pVM);
7218	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7219	if (RT_LIKELY(rc == VINF_SUCCESS))
7220	{
7221	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7222	Assert(pVmxTransient->cbInstr == 3);
7223	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7224	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7225	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7226	}
7227	else
7228	{
7229	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7230	rc = VERR_EM_INTERPRETER;
7231	}
7232	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7233	return rc;
7234	}
7235
7236
7237	/**
7238	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7239	*/
7240	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7241	{
7242	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7243	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7244	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7245	AssertRCReturn(rc, rc);
7246
7247	PVM pVM = pVCpu->CTX_SUFF(pVM);
7248	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7249	if (RT_LIKELY(rc == VINF_SUCCESS))
7250	{
7251	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7252	Assert(pVmxTransient->cbInstr == 2);
7253	}
7254	else
7255	{
7256	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7257	rc = VERR_EM_INTERPRETER;
7258	}
7259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7260	return rc;
7261	}
7262
7263
7264	/**
7265	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7266	*/
7267	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7268	{
7269	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7270	PVM pVM = pVCpu->CTX_SUFF(pVM);
7271	Assert(!pVM->hm.s.fNestedPaging);
7272
7273	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7274	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7275	AssertRCReturn(rc, rc);
7276
7277	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7278	rc = VBOXSTRICTRC_VAL(rc2);
7279	if (RT_LIKELY(rc == VINF_SUCCESS))
7280	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7281	else
7282	{
7283	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7284	pVmxTransient->uExitQualification, rc));
7285	}
7286	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7287	return rc;
7288	}
7289
7290
7291	/**
7292	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7293	*/
7294	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7295	{
7296	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7297	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7298	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7299	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7300	AssertRCReturn(rc, rc);
7301
7302	PVM pVM = pVCpu->CTX_SUFF(pVM);
7303	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7304	if (RT_LIKELY(rc == VINF_SUCCESS))
7305	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7306	else
7307	{
7308	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7309	rc = VERR_EM_INTERPRETER;
7310	}
7311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7312	return rc;
7313	}
7314
7315
7316	/**
7317	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7318	*/
7319	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7320	{
7321	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7322	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7323	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7324	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7325	AssertRCReturn(rc, rc);
7326
7327	PVM pVM = pVCpu->CTX_SUFF(pVM);
7328	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7329	rc = VBOXSTRICTRC_VAL(rc2);
7330	if (RT_LIKELY( rc == VINF_SUCCESS
7331	\|\| rc == VINF_EM_HALT))
7332	{
7333	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7334	AssertRCReturn(rc3, rc3);
7335
7336	if ( rc == VINF_EM_HALT
7337	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7338	{
7339	rc = VINF_SUCCESS;
7340	}
7341	}
7342	else
7343	{
7344	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7345	rc = VERR_EM_INTERPRETER;
7346	}
7347	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7348	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7350	return rc;
7351	}
7352
7353
7354	/**
7355	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7356	*/
7357	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7358	{
7359	/*
7360	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7361	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7362	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7363	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7364	*/
7365	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7366	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7367	}
7368
7369
7370	/**
7371	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7372	*/
7373	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7374	{
7375	/*
7376	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7377	* root operation. If we get there there is something funny going on.
7378	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7379	*/
7380	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7381	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7382	}
7383
7384
7385	/**
7386	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7387	*/
7388	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7389	{
7390	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7391	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7392	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7393	}
7394
7395
7396	/**
7397	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7398	*/
7399	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7400	{
7401	/*
7402	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7403	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7404	* See Intel spec. 25.3 "Other Causes of VM-exits".
7405	*/
7406	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7407	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7408	}
7409
7410
7411	/**
7412	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7413	* VM-exit.
7414	*/
7415	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7416	{
7417	/*
7418	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7419	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7420	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7421	*/
7422	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7423	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7424	}
7425
7426
7427	/**
7428	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7429	* VM-exit.
7430	*/
7431	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7432	{
7433	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7434	return VINF_EM_RESET;
7435	}
7436
7437
7438	/**
7439	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7440	*/
7441	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7442	{
7443	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7444	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7445	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7446	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7447	AssertRCReturn(rc, rc);
7448
7449	pMixedCtx->rip++;
7450	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7451	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7452	rc = VINF_SUCCESS;
7453	else
7454	rc = VINF_EM_HALT;
7455
7456	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7457	return rc;
7458	}
7459
7460
7461	/**
7462	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7463	*/
7464	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7465	{
7466	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7467	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7468	return VINF_SUCCESS;
7469	}
7470
7471
7472	/**
7473	* VM-exit handler for expiry of the VMX preemption timer.
7474	*/
7475	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7476	{
7477	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7478
7479	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7480	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7481
7482	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7483	PVM pVM = pVCpu->CTX_SUFF(pVM);
7484	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7486	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7487	}
7488
7489
7490	/**
7491	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7492	*/
7493	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7494	{
7495	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7496	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7497	/** @todo check if XSETBV is supported by the recompiler. */
7498	return VERR_EM_INTERPRETER;
7499	}
7500
7501
7502	/**
7503	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7504	*/
7505	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7506	{
7507	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7508	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7509	/** @todo implement EMInterpretInvpcid() */
7510	return VERR_EM_INTERPRETER;
7511	}
7512
7513
7514	/**
7515	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7516	* Error VM-exit.
7517	*/
7518	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7519	{
7520	uint32_t uIntrState;
7521	HMVMXHCUINTREG uHCReg;
7522	uint64_t u64Val;
7523	uint32_t u32Val;
7524
7525	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7526	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7527	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7528	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7529	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7530	AssertRCReturn(rc, rc);
7531
7532	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7533	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7534	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7535	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7536
7537	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7538	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7539	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7540	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7541	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7542	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7543	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7544	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7545	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7546	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7547	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7548	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7549
7550	PVM pVM = pVCpu->CTX_SUFF(pVM);
7551	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7552
7553	return VERR_VMX_INVALID_GUEST_STATE;
7554	}
7555
7556
7557	/**
7558	* VM-exit handler for VM-entry failure due to an MSR-load
7559	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7560	*/
7561	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7562	{
7563	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7564	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7565	}
7566
7567
7568	/**
7569	* VM-exit handler for VM-entry failure due to a machine-check event
7570	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7571	*/
7572	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7573	{
7574	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7575	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7576	}
7577
7578
7579	/**
7580	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7581	* theory.
7582	*/
7583	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7584	{
7585	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7586	return VERR_VMX_UNDEFINED_EXIT_CODE;
7587	}
7588
7589
7590	/**
7591	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7592	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7593	* Conditional VM-exit.
7594	*/
7595	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7596	{
7597	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7598	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7600	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7601	return VERR_EM_INTERPRETER;
7602	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7603	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7604	}
7605
7606
7607	/**
7608	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7609	*/
7610	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7611	{
7612	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7613	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7614	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7615	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7616	return VERR_EM_INTERPRETER;
7617	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7618	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7619	}
7620
7621
7622	/**
7623	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7624	*/
7625	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7626	{
7627	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7628	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7629	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7630	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7631	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7632	AssertRCReturn(rc, rc);
7633
7634	PVM pVM = pVCpu->CTX_SUFF(pVM);
7635	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7636	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7637	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7638	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7639
7640	if (RT_LIKELY(rc == VINF_SUCCESS))
7641	{
7642	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7643	Assert(pVmxTransient->cbInstr == 2);
7644	}
7645	return rc;
7646	}
7647
7648
7649	/**
7650	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7651	*/
7652	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7653	{
7654	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7655	PVM pVM = pVCpu->CTX_SUFF(pVM);
7656	int rc = VINF_SUCCESS;
7657
7658	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7659	if ( pVM->hm.s.fTPRPatchingActive
7660	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7661	{
7662	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7663	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7664	{
7665	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7666	AssertRC(rc);
7667	}
7668
7669	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7670	Assert(pVmxTransient->cbInstr == 2);
7671	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7672	return VINF_SUCCESS;
7673	}
7674
7675	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7676	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7677	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7678	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7679	AssertRCReturn(rc, rc);
7680	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7681
7682	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7683	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7685
7686	if (RT_LIKELY(rc == VINF_SUCCESS))
7687	{
7688	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7689
7690	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7691	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7692	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7693	{
7694	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7695	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7696	}
7697	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7698	{
7699	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7700	AssertRCReturn(rc, rc);
7701	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7702	}
7703	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7704	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7705
7706	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7707	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7708	{
7709	switch (pMixedCtx->ecx)
7710	{
7711	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7712	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7713	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7714	case MSR_K8_FS_BASE: /* no break */
7715	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7716	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7717	}
7718	}
7719	#ifdef VBOX_STRICT
7720	else
7721	{
7722	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7723	switch (pMixedCtx->ecx)
7724	{
7725	case MSR_IA32_SYSENTER_CS:
7726	case MSR_IA32_SYSENTER_EIP:
7727	case MSR_IA32_SYSENTER_ESP:
7728	case MSR_K8_FS_BASE:
7729	case MSR_K8_GS_BASE:
7730	{
7731	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7732	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7733	}
7734
7735	case MSR_K8_LSTAR:
7736	case MSR_K6_STAR:
7737	case MSR_K8_SF_MASK:
7738	case MSR_K8_TSC_AUX:
7739	case MSR_K8_KERNEL_GS_BASE:
7740	{
7741	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7742	pMixedCtx->ecx));
7743	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7744	}
7745	}
7746	}
7747	#endif /* VBOX_STRICT */
7748	}
7749	return rc;
7750	}
7751
7752
7753	/**
7754	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7755	*/
7756	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7757	{
7758	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7759	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7760	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7761	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7762	return VERR_EM_INTERPRETER;
7763	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7764	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7765	}
7766
7767
7768	/**
7769	* VM-exit handler for when the TPR value is lowered below the specified
7770	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7771	*/
7772	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7773	{
7774	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7775	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7776
7777	/*
7778	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7779	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7780	* resume guest execution.
7781	*/
7782	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7784	return VINF_SUCCESS;
7785	}
7786
7787
7788	/**
7789	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7790	* VM-exit.
7791	*
7792	* @retval VINF_SUCCESS when guest execution can continue.
7793	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7794	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7795	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7796	* recompiler.
7797	*/
7798	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7799	{
7800	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7801	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7802	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7803	AssertRCReturn(rc, rc);
7804
7805	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7806	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7807	PVM pVM = pVCpu->CTX_SUFF(pVM);
7808	switch (uAccessType)
7809	{
7810	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7811	{
7812	#if 0
7813	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7814	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7815	#else
7816	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7817	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7818	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7819	#endif
7820	AssertRCReturn(rc, rc);
7821
7822	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7823	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7824	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7825	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7826
7827	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7828	{
7829	case 0: /* CR0 */
7830	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7831	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7832	break;
7833	case 2: /* C2 **/
7834	/* Nothing to do here, CR2 it's not part of the VMCS. */
7835	break;
7836	case 3: /* CR3 */
7837	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7838	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7839	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7840	break;
7841	case 4: /* CR4 */
7842	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7843	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7844	break;
7845	case 8: /* CR8 */
7846	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7847	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7848	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7849	break;
7850	default:
7851	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7852	break;
7853	}
7854
7855	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7856	break;
7857	}
7858
7859	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7860	{
7861	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7862	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7863	AssertRCReturn(rc, rc);
7864	Assert( !pVM->hm.s.fNestedPaging
7865	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7866	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7867
7868	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7869	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7870	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7871
7872	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7873	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7874	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7875	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7876	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7877	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7878	break;
7879	}
7880
7881	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7882	{
7883	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7884	AssertRCReturn(rc, rc);
7885	rc = EMInterpretCLTS(pVM, pVCpu);
7886	AssertRCReturn(rc, rc);
7887	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7889	Log(("CRX CLTS write rc=%d\n", rc));
7890	break;
7891	}
7892
7893	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7894	{
7895	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7896	AssertRCReturn(rc, rc);
7897	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7898	if (RT_LIKELY(rc == VINF_SUCCESS))
7899	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7901	Log(("CRX LMSW write rc=%d\n", rc));
7902	break;
7903	}
7904
7905	default:
7906	{
7907	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7908	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7909	}
7910	}
7911
7912	/* Validate possible error codes. */
7913	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7914	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7915	if (RT_SUCCESS(rc))
7916	{
7917	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7918	AssertRCReturn(rc2, rc2);
7919	}
7920
7921	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7922	return rc;
7923	}
7924
7925
7926	/**
7927	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7928	* VM-exit.
7929	*/
7930	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7931	{
7932	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7933	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7934
7935	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7936	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7937	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7938	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7939	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7940	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7941	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7942	AssertRCReturn(rc, rc);
7943	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7944
7945	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7946	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7947	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7948	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7949	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7950	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7951	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7952
7953	/* I/O operation lookup arrays. */
7954	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7955	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7956
7957	const uint32_t cbSize = s_aIOSize[uIOWidth];
7958	const uint32_t cbInstr = pVmxTransient->cbInstr;
7959	PVM pVM = pVCpu->CTX_SUFF(pVM);
7960	if (fIOString)
7961	{
7962	/* INS/OUTS - I/O String instruction. */
7963	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7964	/** @todo for now manually disassemble later optimize by getting the fields from
7965	* the VMCS. */
7966	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7967	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7968	* segment prefix info. */
7969	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7970	if (RT_SUCCESS(rc))
7971	{
7972	if (fIOWrite)
7973	{
7974	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7975	(DISCPUMODE)pDis->uAddrMode, cbSize);
7976	rc = VBOXSTRICTRC_VAL(rc2);
7977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7978	}
7979	else
7980	{
7981	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7982	(DISCPUMODE)pDis->uAddrMode, cbSize);
7983	rc = VBOXSTRICTRC_VAL(rc2);
7984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7985	}
7986	}
7987	else
7988	{
7989	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7990	rc = VINF_EM_RAW_EMULATE_INSTR;
7991	}
7992	}
7993	else
7994	{
7995	/* IN/OUT - I/O instruction. */
7996	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7997	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7998	if (fIOWrite)
7999	{
8000	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8001	rc = VBOXSTRICTRC_VAL(rc2);
8002	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8003	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8004	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8005	}
8006	else
8007	{
8008	uint32_t u32Result = 0;
8009	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8010	rc = VBOXSTRICTRC_VAL(rc2);
8011	if (IOM_SUCCESS(rc))
8012	{
8013	/* Save result of I/O IN instr. in AL/AX/EAX. */
8014	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8015	}
8016	else if (rc == VINF_IOM_R3_IOPORT_READ)
8017	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8019	}
8020	}
8021
8022	if (IOM_SUCCESS(rc))
8023	{
8024	pMixedCtx->rip += cbInstr;
8025	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8026	if (RT_LIKELY(rc == VINF_SUCCESS))
8027	{
8028	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8029	AssertRCReturn(rc, rc);
8030
8031	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8032	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8033	{
8034	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8035	for (unsigned i = 0; i < 4; i++)
8036	{
8037	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8038	if ( ( uIOPort >= pMixedCtx->dr[i]
8039	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8040	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8041	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8042	{
8043	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8044	uint64_t uDR6 = ASMGetDR6();
8045
8046	/* Clear all breakpoint status flags and set the one we just hit. */
8047	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8048	uDR6 \|= (uint64_t)RT_BIT(i);
8049
8050	/*
8051	* Note: AMD64 Architecture Programmer's Manual 13.1:
8052	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8053	* be cleared by software after the contents have been read.
8054	*/
8055	ASMSetDR6(uDR6);
8056
8057	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8058	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8059
8060	/* Paranoia. */
8061	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8062	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8063	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8064
8065	/* Resync DR7 */
8066	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8067	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8068
8069	/* Set #DB to be injected into the VM and continue guest execution. */
8070	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8071	break;
8072	}
8073	}
8074	}
8075	}
8076	}
8077
8078	#ifdef DEBUG
8079	if (rc == VINF_IOM_R3_IOPORT_READ)
8080	Assert(!fIOWrite);
8081	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8082	Assert(fIOWrite);
8083	else
8084	{
8085	AssertMsg( RT_FAILURE(rc)
8086	\|\| rc == VINF_SUCCESS
8087	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8088	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8089	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8090	}
8091	#endif
8092
8093	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8094	return rc;
8095	}
8096
8097
8098	/**
8099	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8100	* VM-exit.
8101	*/
8102	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8103	{
8104	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8105
8106	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8107	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8108	AssertRCReturn(rc, rc);
8109	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8110	{
8111	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8112	AssertRCReturn(rc, rc);
8113	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8114	{
8115	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8116	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8117	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8118	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8119	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8120	{
8121	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8122	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8123
8124	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8125	Assert(!pVCpu->hm.s.Event.fPending);
8126	pVCpu->hm.s.Event.fPending = true;
8127	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8128	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8129	AssertRCReturn(rc, rc);
8130	if (fErrorCodeValid)
8131	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8132	else
8133	pVCpu->hm.s.Event.u32ErrCode = 0;
8134	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8135	&& uVector == X86_XCPT_PF)
8136	{
8137	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8138	}
8139	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8140	}
8141	}
8142	}
8143	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8144	* emulation. */
8145	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8146	return VERR_EM_INTERPRETER;
8147	}
8148
8149
8150	/**
8151	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8152	*/
8153	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8154	{
8155	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8156	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8157	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8158	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8159	AssertRCReturn(rc, rc);
8160	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8161	return VINF_EM_DBG_STOP;
8162	}
8163
8164
8165	/**
8166	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8167	*/
8168	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8169	{
8170	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8171
8172	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8173	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8174	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8175	return VINF_SUCCESS;
8176	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8177	return rc;
8178
8179	#if 0
8180	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8181	* just sync the whole thing. */
8182	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8183	#else
8184	/* Aggressive state sync. for now. */
8185	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8186	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8187	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8188	#endif
8189	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8190	AssertRCReturn(rc, rc);
8191
8192	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8193	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8194	switch (uAccessType)
8195	{
8196	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8197	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8198	{
8199	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8200	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8201	{
8202	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8203	}
8204
8205	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8206	GCPhys &= PAGE_BASE_GC_MASK;
8207	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8208	PVM pVM = pVCpu->CTX_SUFF(pVM);
8209	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8210	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8211
8212	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8213	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8214	CPUMCTX2CORE(pMixedCtx), GCPhys);
8215	rc = VBOXSTRICTRC_VAL(rc2);
8216	Log(("ApicAccess rc=%d\n", rc));
8217	if ( rc == VINF_SUCCESS
8218	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8219	\|\| rc == VERR_PAGE_NOT_PRESENT)
8220	{
8221	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8222	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8223	rc = VINF_SUCCESS;
8224	}
8225	break;
8226	}
8227
8228	default:
8229	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8230	rc = VINF_EM_RAW_EMULATE_INSTR;
8231	break;
8232	}
8233
8234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8235	return rc;
8236	}
8237
8238
8239	/**
8240	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8241	* VM-exit.
8242	*/
8243	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8244	{
8245	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8246
8247	/* We should -not- get this VM-exit if the guest is debugging. */
8248	if (CPUMIsGuestDebugStateActive(pVCpu))
8249	{
8250	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8251	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8252	}
8253
8254	int rc = VERR_INTERNAL_ERROR_5;
8255	if ( !DBGFIsStepping(pVCpu)
8256	&& !CPUMIsHyperDebugStateActive(pVCpu))
8257	{
8258	/* Don't intercept MOV DRx. */
8259	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8260	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8261	AssertRCReturn(rc, rc);
8262
8263	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8264	PVM pVM = pVCpu->CTX_SUFF(pVM);
8265	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8266	AssertRC(rc);
8267	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8268
8269	#ifdef VBOX_WITH_STATISTICS
8270	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8271	AssertRCReturn(rc, rc);
8272	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8274	else
8275	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8276	#endif
8277	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8278	return VINF_SUCCESS;
8279	}
8280
8281	/*
8282	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8283	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8284	*/
8285	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8286	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8287	AssertRCReturn(rc, rc);
8288
8289	PVM pVM = pVCpu->CTX_SUFF(pVM);
8290	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8291	{
8292	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8293	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8294	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8295	if (RT_SUCCESS(rc))
8296	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8298	}
8299	else
8300	{
8301	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8302	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8303	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8305	}
8306
8307	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8308	if (RT_SUCCESS(rc))
8309	{
8310	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8311	AssertRCReturn(rc2, rc2);
8312	}
8313	return rc;
8314	}
8315
8316
8317	/**
8318	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8319	* Conditional VM-exit.
8320	*/
8321	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8322	{
8323	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8324	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8325
8326	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8327	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8328	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8329	return VINF_SUCCESS;
8330	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8331	return rc;
8332
8333	RTGCPHYS GCPhys = 0;
8334	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8335
8336	#if 0
8337	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8338	#else
8339	/* Aggressive state sync. for now. */
8340	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8341	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8342	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8343	#endif
8344	AssertRCReturn(rc, rc);
8345
8346	/*
8347	* If we succeed, resume guest execution.
8348	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8349	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8350	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8351	* weird case. See @bugref{6043}.
8352	*/
8353	PVM pVM = pVCpu->CTX_SUFF(pVM);
8354	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8355	rc = VBOXSTRICTRC_VAL(rc2);
8356	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8357	if ( rc == VINF_SUCCESS
8358	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8359	\|\| rc == VERR_PAGE_NOT_PRESENT)
8360	{
8361	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8362	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8363	return VINF_SUCCESS;
8364	}
8365	return rc;
8366	}
8367
8368
8369	/**
8370	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8371	* VM-exit.
8372	*/
8373	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8374	{
8375	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8376	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8377
8378	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8379	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8380	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8381	return VINF_SUCCESS;
8382	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8383	return rc;
8384
8385	RTGCPHYS GCPhys = 0;
8386	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8387	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8388	#if 0
8389	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8390	#else
8391	/* Aggressive state sync. for now. */
8392	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8393	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8394	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8395	#endif
8396	AssertRCReturn(rc, rc);
8397
8398	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8399	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8400
8401	RTGCUINT uErrorCode = 0;
8402	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8403	uErrorCode \|= X86_TRAP_PF_ID;
8404	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8405	uErrorCode \|= X86_TRAP_PF_RW;
8406	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8407	uErrorCode \|= X86_TRAP_PF_P;
8408
8409	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8410
8411	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8412	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8413
8414	/* Handle the pagefault trap for the nested shadow table. */
8415	PVM pVM = pVCpu->CTX_SUFF(pVM);
8416	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8417	TRPMResetTrap(pVCpu);
8418
8419	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8420	if ( rc == VINF_SUCCESS
8421	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8422	\|\| rc == VERR_PAGE_NOT_PRESENT)
8423	{
8424	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8426	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8427	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8428	return VINF_SUCCESS;
8429	}
8430
8431	Log(("EPT return to ring-3 rc=%d\n"));
8432	return rc;
8433	}
8434
8435
8436	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8437	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8438	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8439	/**
8440	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8441	*/
8442	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8443	{
8444	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8445	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8446
8447	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8448	AssertRCReturn(rc, rc);
8449
8450	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8451	{
8452	/* Old-style FPU error reporting needs some extra work. */
8453	/** @todo don't fall back to the recompiler, but do it manually. */
8454	return VERR_EM_INTERPRETER;
8455	}
8456	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8457	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8458	return rc;
8459	}
8460
8461
8462	/**
8463	* VM-exit exception handler for #BP (Breakpoint exception).
8464	*/
8465	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8466	{
8467	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8468	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8469
8470	/** @todo Try optimize this by not saving the entire guest state unless
8471	* really needed. */
8472	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8473	AssertRCReturn(rc, rc);
8474
8475	PVM pVM = pVCpu->CTX_SUFF(pVM);
8476	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8477	if (rc == VINF_EM_RAW_GUEST_TRAP)
8478	{
8479	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8480	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8481	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8482	AssertRCReturn(rc, rc);
8483
8484	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8485	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8486	}
8487
8488	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8489	return rc;
8490	}
8491
8492
8493	/**
8494	* VM-exit exception handler for #DB (Debug exception).
8495	*/
8496	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8497	{
8498	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8499	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8500
8501	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8502	AssertRCReturn(rc, rc);
8503
8504	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8505	uint64_t uDR6 = X86_DR6_INIT_VAL;
8506	uDR6 \|= (pVmxTransient->uExitQualification
8507	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8508	PVM pVM = pVCpu->CTX_SUFF(pVM);
8509	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8510	if (rc == VINF_EM_RAW_GUEST_TRAP)
8511	{
8512	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8513	pMixedCtx->dr[6] = uDR6;
8514
8515	if (CPUMIsGuestDebugStateActive(pVCpu))
8516	ASMSetDR6(pMixedCtx->dr[6]);
8517
8518	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8519
8520	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8521	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8522
8523	/* Paranoia. */
8524	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8525	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8526	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8527
8528	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8529	AssertRCReturn(rc,rc);
8530
8531	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8532	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8533	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8534	AssertRCReturn(rc2, rc2);
8535	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8536	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8537	rc = VINF_SUCCESS;
8538	}
8539
8540	return rc;
8541	}
8542
8543
8544	/**
8545	* VM-exit exception handler for #NM (Device-not-available exception: floating
8546	* point exception).
8547	*/
8548	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8549	{
8550	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8551
8552	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8553	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8554	#endif
8555
8556	/* We require CR0 and EFER. EFER is always up-to-date. */
8557	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8558	AssertRCReturn(rc, rc);
8559
8560	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8561	PVM pVM = pVCpu->CTX_SUFF(pVM);
8562	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8563	if (rc == VINF_SUCCESS)
8564	{
8565	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8566	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8568	return VINF_SUCCESS;
8569	}
8570
8571	/* Forward #NM to the guest. */
8572	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8573	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8574	AssertRCReturn(rc, rc);
8575	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8576	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8578	return rc;
8579	}
8580
8581
8582	/**
8583	* VM-exit exception handler for #GP (General-protection exception).
8584	*
8585	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8586	*/
8587	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8588	{
8589	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8591
8592	int rc = VERR_INTERNAL_ERROR_5;
8593	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8594	{
8595	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8596	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8597	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8598	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8599	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8600	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8601	AssertRCReturn(rc, rc);
8602	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8603	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8604	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8605	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8606	return rc;
8607	#else
8608	/* We don't intercept #GP. */
8609	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8610	return VERR_VMX_UNEXPECTED_EXCEPTION;
8611	#endif
8612	}
8613
8614	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8615	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8616
8617	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8618	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8619	AssertRCReturn(rc, rc);
8620
8621	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8622	uint32_t cbOp = 0;
8623	PVM pVM = pVCpu->CTX_SUFF(pVM);
8624	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8625	if (RT_SUCCESS(rc))
8626	{
8627	rc = VINF_SUCCESS;
8628	Assert(cbOp == pDis->cbInstr);
8629	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8630	switch (pDis->pCurInstr->uOpcode)
8631	{
8632	case OP_CLI:
8633	pMixedCtx->eflags.Bits.u1IF = 0;
8634	pMixedCtx->rip += pDis->cbInstr;
8635	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8636	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8637	break;
8638
8639	case OP_STI:
8640	pMixedCtx->eflags.Bits.u1IF = 1;
8641	pMixedCtx->rip += pDis->cbInstr;
8642	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8643	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8644	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8645	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8646	break;
8647
8648	case OP_HLT:
8649	rc = VINF_EM_HALT;
8650	pMixedCtx->rip += pDis->cbInstr;
8651	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8653	break;
8654
8655	case OP_POPF:
8656	{
8657	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8658	uint32_t cbParm = 0;
8659	uint32_t uMask = 0;
8660	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8661	{
8662	cbParm = 4;
8663	uMask = 0xffffffff;
8664	}
8665	else
8666	{
8667	cbParm = 2;
8668	uMask = 0xffff;
8669	}
8670
8671	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8672	RTGCPTR GCPtrStack = 0;
8673	X86EFLAGS uEflags;
8674	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8675	&GCPtrStack);
8676	if (RT_SUCCESS(rc))
8677	{
8678	Assert(sizeof(uEflags.u32) >= cbParm);
8679	uEflags.u32 = 0;
8680	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8681	}
8682	if (RT_FAILURE(rc))
8683	{
8684	rc = VERR_EM_INTERPRETER;
8685	break;
8686	}
8687	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8688	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8689	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8690	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8691	pMixedCtx->eflags.Bits.u1RF = 0;
8692	pMixedCtx->esp += cbParm;
8693	pMixedCtx->esp &= uMask;
8694	pMixedCtx->rip += pDis->cbInstr;
8695	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8697	break;
8698	}
8699
8700	case OP_PUSHF:
8701	{
8702	uint32_t cbParm = 0;
8703	uint32_t uMask = 0;
8704	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8705	{
8706	cbParm = 4;
8707	uMask = 0xffffffff;
8708	}
8709	else
8710	{
8711	cbParm = 2;
8712	uMask = 0xffff;
8713	}
8714
8715	/* Get the stack pointer & push the contents of eflags onto the stack. */
8716	RTGCPTR GCPtrStack = 0;
8717	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8718	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8719	if (RT_FAILURE(rc))
8720	{
8721	rc = VERR_EM_INTERPRETER;
8722	break;
8723	}
8724	X86EFLAGS uEflags;
8725	uEflags = pMixedCtx->eflags;
8726	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8727	uEflags.Bits.u1RF = 0;
8728	uEflags.Bits.u1VM = 0;
8729
8730	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8731	if (RT_FAILURE(rc))
8732	{
8733	rc = VERR_EM_INTERPRETER;
8734	break;
8735	}
8736	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8737	pMixedCtx->esp -= cbParm;
8738	pMixedCtx->esp &= uMask;
8739	pMixedCtx->rip += pDis->cbInstr;
8740	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8742	break;
8743	}
8744
8745	case OP_IRET:
8746	{
8747	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8748	* instruction reference. */
8749	RTGCPTR GCPtrStack = 0;
8750	uint32_t uMask = 0xffff;
8751	uint16_t aIretFrame[3];
8752	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8753	{
8754	rc = VERR_EM_INTERPRETER;
8755	break;
8756	}
8757	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8758	&GCPtrStack);
8759	if (RT_SUCCESS(rc))
8760	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8761	if (RT_FAILURE(rc))
8762	{
8763	rc = VERR_EM_INTERPRETER;
8764	break;
8765	}
8766	pMixedCtx->eip = 0;
8767	pMixedCtx->ip = aIretFrame[0];
8768	pMixedCtx->cs.Sel = aIretFrame[1];
8769	pMixedCtx->cs.ValidSel = aIretFrame[1];
8770	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8771	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8772	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8773	pMixedCtx->sp += sizeof(aIretFrame);
8774	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8775	\| HM_CHANGED_GUEST_RFLAGS;
8776	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8778	break;
8779	}
8780
8781	case OP_INT:
8782	{
8783	uint16_t uVector = pDis->Param1.uValue & 0xff;
8784	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8785	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8786	break;
8787	}
8788
8789	case OP_INTO:
8790	{
8791	if (pMixedCtx->eflags.Bits.u1OF)
8792	{
8793	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8795	}
8796	break;
8797	}
8798
8799	default:
8800	{
8801	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8802	EMCODETYPE_SUPERVISOR);
8803	rc = VBOXSTRICTRC_VAL(rc2);
8804	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8805	Log(("#GP rc=%Rrc\n", rc));
8806	break;
8807	}
8808	}
8809	}
8810	else
8811	rc = VERR_EM_INTERPRETER;
8812
8813	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8814	("#GP Unexpected rc=%Rrc\n", rc));
8815	return rc;
8816	}
8817
8818
8819	/**
8820	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8821	* the exception reported in the VMX transient structure back into the VM.
8822	*
8823	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8824	* up-to-date.
8825	*/
8826	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8827	{
8828	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8829
8830	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8831	hmR0VmxCheckExitDueToEventDelivery(). */
8832	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8833	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8834	AssertRCReturn(rc, rc);
8835	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8836	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8837	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8838	return VINF_SUCCESS;
8839	}
8840
8841
8842	/**
8843	* VM-exit exception handler for #PF (Page-fault exception).
8844	*/
8845	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8846	{
8847	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8848	PVM pVM = pVCpu->CTX_SUFF(pVM);
8849	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8850	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8851	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8852	AssertRCReturn(rc, rc);
8853
8854	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8855	if (pVM->hm.s.fNestedPaging)
8856	{
8857	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8858	{
8859	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8860	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8861	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8862	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8864	}
8865	else
8866	{
8867	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8868	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8869	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8870	Log(("Pending #DF due to vectoring #PF. NP\n"));
8871	}
8872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8873	return rc;
8874	}
8875	#else
8876	Assert(!pVM->hm.s.fNestedPaging);
8877	#endif
8878
8879	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8880	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8881	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8882	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8883	AssertRCReturn(rc, rc);
8884	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8885	if ( pVM->hm.s.fTRPPatchingAllowed
8886	&& pVM->hm.s.pGuestPatchMem
8887	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8888	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8889	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8890	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8891	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8892	{
8893	RTGCPHYS GCPhys;
8894	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8895	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8896	if ( rc == VINF_SUCCESS
8897	&& GCPhys == GCPhysApicBase)
8898	{
8899	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8900	AssertRCReturn(rc, rc);
8901
8902	/* Only attempt to patch the instruction once. */
8903	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8904	if (!pPatch)
8905	return VINF_EM_HM_PATCH_TPR_INSTR;
8906	}
8907	}
8908	#endif
8909
8910	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8911	AssertRCReturn(rc, rc);
8912
8913	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8914	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8915
8916	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8917	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8918	(RTGCPTR)pVmxTransient->uExitQualification);
8919
8920	Log(("#PF: rc=%Rrc\n", rc));
8921	if (rc == VINF_SUCCESS)
8922	{
8923	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8924	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8925	* memory? We don't update the whole state here... */
8926	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8927	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8928	TRPMResetTrap(pVCpu);
8929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8930	return rc;
8931	}
8932	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8933	{
8934	if (!pVmxTransient->fVectoringPF)
8935	{
8936	/* It's a guest page fault and needs to be reflected to the guest. */
8937	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8938	TRPMResetTrap(pVCpu);
8939	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8940	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8941	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8942	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8943	}
8944	else
8945	{
8946	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8947	TRPMResetTrap(pVCpu);
8948	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8949	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8950	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8951	}
8952
8953	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8954	return VINF_SUCCESS;
8955	}
8956
8957	TRPMResetTrap(pVCpu);
8958	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8959	return rc;
8960	}
8961

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45892

Download in other formats: