HMVMXR0.cpp@ 45918

Last change on this file since 45918 was 45918, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: State-save ordering enforcement assertion, comment and don't save autoload/store MSRs for getting EFER as it's always up to date.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 378.7 KB

Line
1	/* $Id: HMVMXR0.cpp 45918 2013-05-06 15:48:04Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147	/** Profiling macro. */
148	#ifdef HM_PROFILE_EXIT_DISPATCH
149	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
150	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
151	#else
152	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
153	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
154	#endif
155
156
157	/*******************************************************************************
158	* Structures and Typedefs *
159	*******************************************************************************/
160	/**
161	* A state structure for holding miscellaneous information across
162	* VMX non-root operation and restored after the transition.
163	*/
164	typedef struct VMXTRANSIENT
165	{
166	/** The host's rflags/eflags. */
167	RTCCUINTREG uEFlags;
168	#if HC_ARCH_BITS == 32
169	uint32_t u32Alignment0;
170	#endif
171	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
172	uint64_t u64LStarMsr;
173	/** The guest's TPR value used for TPR shadowing. */
174	uint8_t u8GuestTpr;
175	/** Alignment. */
176	uint8_t abAlignment0[6];
177
178	/** The basic VM-exit reason. */
179	uint16_t uExitReason;
180	/** Alignment. */
181	uint16_t u16Alignment0;
182	/** The VM-exit interruption error code. */
183	uint32_t uExitIntrErrorCode;
184	/** The VM-exit exit qualification. */
185	RTGCUINTPTR uExitQualification;
186	#if GC_ARCH_BITS == 32
187	/** Alignment. */
188	uint32_t u32Alignment1;
189	#endif
190
191	/** The VM-exit interruption-information field. */
192	uint32_t uExitIntrInfo;
193	/** The VM-exit instruction-length field. */
194	uint32_t cbInstr;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Alignment. */
198	uint8_t abAlignment1[5];
199
200	/** The VM-entry interruption-information field. */
201	uint32_t uEntryIntrInfo;
202	/** The VM-entry exception error code field. */
203	uint32_t uEntryXcptErrorCode;
204	/** The VM-entry instruction length field. */
205	uint32_t cbEntryInstr;
206
207	/** IDT-vectoring information field. */
208	uint32_t uIdtVectoringInfo;
209	/** IDT-vectoring error code. */
210	uint32_t uIdtVectoringErrorCode;
211
212	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
213	uint32_t fVmcsFieldsRead;
214	/** Whether TSC-offsetting should be setup before VM-entry. */
215	bool fUpdateTscOffsettingAndPreemptTimer;
216	/** Whether the VM-exit was caused by a page-fault during delivery of a
217	* contributary exception or a page-fault. */
218	bool fVectoringPF;
219	} VMXTRANSIENT, *PVMXTRANSIENT;
220	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
221	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
222	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
223
224
225	/**
226	* MSR-bitmap read permissions.
227	*/
228	typedef enum VMXMSREXITREAD
229	{
230	/** Reading this MSR causes a VM-exit. */
231	VMXMSREXIT_INTERCEPT_READ = 0xb,
232	/** Reading this MSR does not cause a VM-exit. */
233	VMXMSREXIT_PASSTHRU_READ
234	} VMXMSREXITREAD;
235
236	/**
237	* MSR-bitmap write permissions.
238	*/
239	typedef enum VMXMSREXITWRITE
240	{
241	/** Writing to this MSR causes a VM-exit. */
242	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
243	/** Writing to this MSR does not cause a VM-exit. */
244	VMXMSREXIT_PASSTHRU_WRITE
245	} VMXMSREXITWRITE;
246
247
248	/*******************************************************************************
249	* Internal Functions *
250	*******************************************************************************/
251	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
252	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
253	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
254	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
255	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
256	#endif
257	#ifndef HMVMX_USE_FUNCTION_TABLE
258	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
259	#define HMVMX_EXIT_DECL static int
260	#else
261	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
262	#endif
263
264	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308
309	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316
317
318	/*******************************************************************************
319	* Global Variables *
320	*******************************************************************************/
321	#ifdef HMVMX_USE_FUNCTION_TABLE
322	/**
323	* VM-exit handler.
324	*
325	* @returns VBox status code.
326	* @param pVCpu Pointer to the VMCPU.
327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
328	* out-of-sync. Make sure to update the required
329	* fields before using them.
330	* @param pVmxTransient Pointer to the VMX-transient structure.
331	*/
332	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333	/** Pointer to VM-exit handler. */
334	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
335
336	/**
337	* VMX_EXIT dispatch table.
338	*/
339	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
340	{
341	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
342	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
343	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
344	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
345	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
346	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
347	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
348	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
349	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
350	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
351	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
352	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
353	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
354	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
355	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
356	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
357	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
358	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
359	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
360	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
361	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
362	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
363	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
364	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
365	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
366	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
367	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
368	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
369	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
370	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
371	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
372	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
373	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
374	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
375	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
376	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
377	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
378	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
379	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
380	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
381	/* 40 UNDEFINED */ hmR0VmxExitPause,
382	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
383	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
384	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
385	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
386	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
387	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
388	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
389	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
390	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
391	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
392	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
393	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
394	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
395	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
396	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
397	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
398	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
399	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
400	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
401	};
402	#endif /* HMVMX_USE_FUNCTION_TABLE */
403
404	#ifdef VBOX_STRICT
405	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
406	{
407	/* 0 */ "(Not Used)",
408	/* 1 */ "VMCALL executed in VMX root operation.",
409	/* 2 */ "VMCLEAR with invalid physical address.",
410	/* 3 */ "VMCLEAR with VMXON pointer.",
411	/* 4 */ "VMLAUNCH with non-clear VMCS.",
412	/* 5 */ "VMRESUME with non-launched VMCS.",
413	/* 6 */ "VMRESUME after VMXOFF",
414	/* 7 */ "VM entry with invalid control fields.",
415	/* 8 */ "VM entry with invalid host state fields.",
416	/* 9 */ "VMPTRLD with invalid physical address.",
417	/* 10 */ "VMPTRLD with VMXON pointer.",
418	/* 11 */ "VMPTRLD with incorrect revision identifier.",
419	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
420	/* 13 */ "VMWRITE to read-only VMCS component.",
421	/* 14 */ "(Not Used)",
422	/* 15 */ "VMXON executed in VMX root operation.",
423	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
424	/* 17 */ "VM entry with non-launched executing VMCS.",
425	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
426	/* 19 */ "VMCALL with non-clear VMCS.",
427	/* 20 */ "VMCALL with invalid VM-exit control fields.",
428	/* 21 */ "(Not Used)",
429	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
430	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
431	/* 24 */ "VMCALL with invalid SMM-monitor features.",
432	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
433	/* 26 */ "VM entry with events blocked by MOV SS.",
434	/* 27 */ "(Not Used)",
435	/* 28 */ "Invalid operand to INVEPT/INVVPID."
436	};
437	#endif /* VBOX_STRICT */
438
439
440
441	/**
442	* Updates the VM's last error record. If there was a VMX instruction error,
443	* reads the error data from the VMCS and updates VCPU's last error record as
444	* well.
445	*
446	* @param pVM Pointer to the VM.
447	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
448	* VERR_VMX_UNABLE_TO_START_VM or
449	* VERR_VMX_INVALID_VMCS_FIELD).
450	* @param rc The error code.
451	*/
452	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
453	{
454	AssertPtr(pVM);
455	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
456	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
457	{
458	AssertPtrReturnVoid(pVCpu);
459	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
460	}
461	pVM->hm.s.lLastError = rc;
462	}
463
464
465	/**
466	* Reads the VM-entry interruption-information field from the VMCS into the VMX
467	* transient structure.
468	*
469	* @returns VBox status code.
470	* @param pVmxTransient Pointer to the VMX transient structure.
471	*
472	* @remarks No-long-jump zone!!!
473	*/
474	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
475	{
476	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
477	AssertRCReturn(rc, rc);
478	return VINF_SUCCESS;
479	}
480
481
482	/**
483	* Reads the VM-entry exception error code field from the VMCS into
484	* the VMX transient structure.
485	*
486	* @returns VBox status code.
487	* @param pVmxTransient Pointer to the VMX transient structure.
488	*
489	* @remarks No-long-jump zone!!!
490	*/
491	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
492	{
493	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
494	AssertRCReturn(rc, rc);
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* Reads the VM-entry exception error code field from the VMCS into
501	* the VMX transient structure.
502	*
503	* @returns VBox status code.
504	* @param pVCpu Pointer to the VMCPU.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-exit interruption-information field from the VMCS into the VMX
519	* transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
528	{
529	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the VM-exit interruption error code from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVCpu Pointer to the VMCPU.
543	* @param pVmxTransient Pointer to the VMX transient structure.
544	*/
545	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
546	{
547	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
548	{
549	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
550	AssertRCReturn(rc, rc);
551	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
552	}
553	return VINF_SUCCESS;
554	}
555
556
557	/**
558	* Reads the VM-exit instruction length field from the VMCS into the VMX
559	* transient structure.
560	*
561	* @returns VBox status code.
562	* @param pVCpu Pointer to the VMCPU.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Reads the exit qualification from the VMCS into the VMX transient structure.
579	*
580	* @returns VBox status code.
581	* @param pVCpu Pointer to the VMCPU.
582	* @param pVmxTransient Pointer to the VMX transient structure.
583	*/
584	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
585	{
586	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
587	{
588	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
589	AssertRCReturn(rc, rc);
590	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
591	}
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the IDT-vectoring information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*
603	* @remarks No-long-jump zone!!!
604	*/
605	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
606	{
607	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
608	{
609	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
610	AssertRCReturn(rc, rc);
611	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
612	}
613	return VINF_SUCCESS;
614	}
615
616
617	/**
618	* Reads the IDT-vectoring error code from the VMCS into the VMX
619	* transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*/
624	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
625	{
626	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
629	AssertRCReturn(rc, rc);
630	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
631	}
632	return VINF_SUCCESS;
633	}
634
635
636	/**
637	* Enters VMX root mode operation on the current CPU.
638	*
639	* @returns VBox status code.
640	* @param pVM Pointer to the VM (optional, can be NULL, after
641	* a resume).
642	* @param HCPhysCpuPage Physical address of the VMXON region.
643	* @param pvCpuPage Pointer to the VMXON region.
644	*/
645	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
646	{
647	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
648	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
649	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
650
651	if (pVM)
652	{
653	/* Write the VMCS revision dword to the VMXON region. */
654	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
655	}
656
657	/* Enable the VMX bit in CR4 if necessary. */
658	RTCCUINTREG uCr4 = ASMGetCR4();
659	if (!(uCr4 & X86_CR4_VMXE))
660	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
661
662	/* Enter VMX root mode. */
663	int rc = VMXEnable(HCPhysCpuPage);
664	if (RT_FAILURE(rc))
665	ASMSetCR4(uCr4);
666
667	return rc;
668	}
669
670
671	/**
672	* Exits VMX root mode operation on the current CPU.
673	*
674	* @returns VBox status code.
675	*/
676	static int hmR0VmxLeaveRootMode(void)
677	{
678	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
679
680	/* If we're for some reason not in VMX root mode, then don't leave it. */
681	if (ASMGetCR4() & X86_CR4_VMXE)
682	{
683	/* Exit VMX root mode and clear the VMX bit in CR4 */
684	VMXDisable();
685	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
686	return VINF_SUCCESS;
687	}
688
689	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
690	}
691
692
693	/**
694	* Allocates and maps one physically contiguous page. The allocated page is
695	* zero'd out. (Used by various VT-x structures).
696	*
697	* @returns IPRT status code.
698	* @param pMemObj Pointer to the ring-0 memory object.
699	* @param ppVirt Where to store the virtual address of the
700	* allocation.
701	* @param pPhys Where to store the physical address of the
702	* allocation.
703	*/
704	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
705	{
706	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
707	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
708	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
709
710	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
711	if (RT_FAILURE(rc))
712	return rc;
713	ppVirt = RTR0MemObjAddress(pMemObj);
714	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
715	ASMMemZero32(*ppVirt, PAGE_SIZE);
716	return VINF_SUCCESS;
717	}
718
719
720	/**
721	* Frees and unmaps an allocated physical page.
722	*
723	* @param pMemObj Pointer to the ring-0 memory object.
724	* @param ppVirt Where to re-initialize the virtual address of
725	* allocation as 0.
726	* @param pHCPhys Where to re-initialize the physical address of the
727	* allocation as 0.
728	*/
729	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
730	{
731	AssertPtr(pMemObj);
732	AssertPtr(ppVirt);
733	AssertPtr(pHCPhys);
734	if (*pMemObj != NIL_RTR0MEMOBJ)
735	{
736	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
737	AssertRC(rc);
738	*pMemObj = NIL_RTR0MEMOBJ;
739	*ppVirt = 0;
740	*pHCPhys = 0;
741	}
742	}
743
744
745	/**
746	* Worker function to free VT-x related structures.
747	*
748	* @returns IPRT status code.
749	* @param pVM Pointer to the VM.
750	*/
751	static void hmR0VmxStructsFree(PVM pVM)
752	{
753	for (VMCPUID i = 0; i < pVM->cCpus; i++)
754	{
755	PVMCPU pVCpu = &pVM->aCpus[i];
756	AssertPtr(pVCpu);
757
758	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
759	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
761	#endif
762
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
765
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
768	}
769
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
772	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
773	#endif
774	}
775
776
777	/**
778	* Worker function to allocate VT-x related VM structures.
779	*
780	* @returns IPRT status code.
781	* @param pVM Pointer to the VM.
782	*/
783	static int hmR0VmxStructsAlloc(PVM pVM)
784	{
785	/*
786	* Initialize members up-front so we can cleanup properly on allocation failure.
787	*/
788	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
789	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
790	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
791	pVM->hm.s.vmx.HCPhys##a_Name = 0;
792
793	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
794	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
795	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
796	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
797
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
800	#endif
801	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
802
803	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
804	for (VMCPUID i = 0; i < pVM->cCpus; i++)
805	{
806	PVMCPU pVCpu = &pVM->aCpus[i];
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
813	#endif
814	}
815	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
816	#undef VMXLOCAL_INIT_VM_MEMOBJ
817
818	/*
819	* Allocate all the VT-x structures.
820	*/
821	int rc = VINF_SUCCESS;
822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
823	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
827	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
828	#endif
829
830	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
831	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
832	{
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
834	&pVM->hm.s.vmx.HCPhysApicAccess);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	/*
840	* Initialize per-VCPU VT-x structures.
841	*/
842	for (VMCPUID i =0; i < pVM->cCpus; i++)
843	{
844	PVMCPU pVCpu = &pVM->aCpus[i];
845	AssertPtr(pVCpu);
846
847	/* Allocate the VM control structure (VMCS). */
848	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
854	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
855	{
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
857	&pVCpu->hm.s.vmx.HCPhysVirtApic);
858	if (RT_FAILURE(rc))
859	goto cleanup;
860	}
861
862	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
864	{
865	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
866	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867	if (RT_FAILURE(rc))
868	goto cleanup;
869	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
870	}
871
872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
873	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
874	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875	if (RT_FAILURE(rc))
876	goto cleanup;
877
878	/* Allocate the VM-exit MSR-load page for the host MSRs. */
879	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	#endif
883	}
884
885	return VINF_SUCCESS;
886
887	cleanup:
888	hmR0VmxStructsFree(pVM);
889	return rc;
890	}
891
892
893	/**
894	* Does global VT-x initialization (called during module initialization).
895	*
896	* @returns VBox status code.
897	*/
898	VMMR0DECL(int) VMXR0GlobalInit(void)
899	{
900	#ifdef HMVMX_USE_FUNCTION_TABLE
901	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
902	# ifdef VBOX_STRICT
903	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
904	Assert(g_apfnVMExitHandlers[i]);
905	# endif
906	#endif
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Does global VT-x termination (called during module termination).
913	*/
914	VMMR0DECL(void) VMXR0GlobalTerm()
915	{
916	/* Nothing to do currently. */
917	}
918
919
920	/**
921	* Sets up and activates VT-x on the current CPU.
922	*
923	* @returns VBox status code.
924	* @param pCpu Pointer to the global CPU info struct.
925	* @param pVM Pointer to the VM (can be NULL after a host resume
926	* operation).
927	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
928	* fEnabledByHost is true).
929	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
930	* @a fEnabledByHost is true).
931	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
932	* enable VT-x/AMD-V on the host.
933	*/
934	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
935	{
936	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
937	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
938
939	if (!fEnabledByHost)
940	{
941	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
942	if (RT_FAILURE(rc))
943	return rc;
944	}
945
946	/*
947	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
948	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
949	* each time while reusing a VPID after hitting the MaxASID limit once.
950	*/
951	if ( pVM
952	&& pVM->hm.s.vmx.fVpid
953	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
954	{
955	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
956	pCpu->fFlushAsidBeforeUse = false;
957	}
958	else
959	pCpu->fFlushAsidBeforeUse = true;
960
961	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
962	++pCpu->cTlbFlushes;
963
964	return VINF_SUCCESS;
965	}
966
967
968	/**
969	* Deactivates VT-x on the current CPU.
970	*
971	* @returns VBox status code.
972	* @param pCpu Pointer to the global CPU info struct.
973	* @param pvCpuPage Pointer to the VMXON region.
974	* @param HCPhysCpuPage Physical address of the VMXON region.
975	*/
976	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
977	{
978	NOREF(pCpu);
979	NOREF(pvCpuPage);
980	NOREF(HCPhysCpuPage);
981
982	return hmR0VmxLeaveRootMode();
983	}
984
985
986	/**
987	* Sets the permission bits for the specified MSR in the MSR bitmap.
988	*
989	* @param pVCpu Pointer to the VMCPU.
990	* @param uMSR The MSR value.
991	* @param enmRead Whether reading this MSR causes a VM-exit.
992	* @param enmWrite Whether writing this MSR causes a VM-exit.
993	*/
994	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
995	{
996	int32_t iBit;
997	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
998
999	/*
1000	* Layout:
1001	* 0x000 - 0x3ff - Low MSR read bits
1002	* 0x400 - 0x7ff - High MSR read bits
1003	* 0x800 - 0xbff - Low MSR write bits
1004	* 0xc00 - 0xfff - High MSR write bits
1005	*/
1006	if (uMsr <= 0x00001FFF)
1007	iBit = uMsr;
1008	else if ( uMsr >= 0xC0000000
1009	&& uMsr <= 0xC0001FFF)
1010	{
1011	iBit = (uMsr - 0xC0000000);
1012	pbMsrBitmap += 0x400;
1013	}
1014	else
1015	{
1016	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1017	return;
1018	}
1019
1020	Assert(iBit <= 0x1fff);
1021	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1022	ASMBitSet(pbMsrBitmap, iBit);
1023	else
1024	ASMBitClear(pbMsrBitmap, iBit);
1025
1026	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1027	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1028	else
1029	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1030	}
1031
1032
1033	/**
1034	* Flushes the TLB using EPT.
1035	*
1036	* @returns VBox status code.
1037	* @param pVM Pointer to the VM.
1038	* @param pVCpu Pointer to the VMCPU.
1039	* @param enmFlush Type of flush.
1040	*/
1041	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1042	{
1043	AssertPtr(pVM);
1044	Assert(pVM->hm.s.fNestedPaging);
1045
1046	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1047
1048	uint64_t descriptor[2];
1049	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1050	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1051
1052	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1053	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1054	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1055	}
1056
1057
1058	/**
1059	* Flushes the TLB using VPID.
1060	*
1061	* @returns VBox status code.
1062	* @param pVM Pointer to the VM.
1063	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1064	* enmFlush).
1065	* @param enmFlush Type of flush.
1066	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1067	* on @a enmFlush).
1068	*/
1069	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1070	{
1071	AssertPtr(pVM);
1072	Assert(pVM->hm.s.vmx.fVpid);
1073
1074	uint64_t descriptor[2];
1075	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1076	{
1077	descriptor[0] = 0;
1078	descriptor[1] = 0;
1079	}
1080	else
1081	{
1082	AssertPtr(pVCpu);
1083	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1084	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1085	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1086	descriptor[1] = GCPtr;
1087	}
1088
1089	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1090	AssertMsg(rc == VINF_SUCCESS,
1091	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1092	if ( RT_SUCCESS(rc)
1093	&& pVCpu)
1094	{
1095	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1096	}
1097	}
1098
1099
1100	/**
1101	* Invalidates a guest page by guest virtual address. Only relevant for
1102	* EPT/VPID, otherwise there is nothing really to invalidate.
1103	*
1104	* @returns VBox status code.
1105	* @param pVM Pointer to the VM.
1106	* @param pVCpu Pointer to the VMCPU.
1107	* @param GCVirt Guest virtual address of the page to invalidate.
1108	*/
1109	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1110	{
1111	AssertPtr(pVM);
1112	AssertPtr(pVCpu);
1113	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1114
1115	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1116	if (!fFlushPending)
1117	{
1118	/*
1119	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1120	* See @bugref{6043} and @bugref{6177}.
1121	*
1122	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1123	* function maybe called in a loop with individual addresses.
1124	*/
1125	if (pVM->hm.s.vmx.fVpid)
1126	{
1127	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1128	{
1129	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1130	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1131	}
1132	else
1133	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1134	}
1135	else if (pVM->hm.s.fNestedPaging)
1136	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1137	}
1138
1139	return VINF_SUCCESS;
1140	}
1141
1142
1143	/**
1144	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1145	* otherwise there is nothing really to invalidate.
1146	*
1147	* @returns VBox status code.
1148	* @param pVM Pointer to the VM.
1149	* @param pVCpu Pointer to the VMCPU.
1150	* @param GCPhys Guest physical address of the page to invalidate.
1151	*/
1152	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1153	{
1154	LogFlowFunc(("%RGp\n", GCPhys));
1155
1156	/*
1157	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1158	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1159	* This function might be called in a loop.
1160	*/
1161	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1163	return VINF_SUCCESS;
1164	}
1165
1166
1167	/**
1168	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1169	* case where neither EPT nor VPID is supported by the CPU.
1170	*
1171	* @param pVM Pointer to the VM.
1172	* @param pVCpu Pointer to the VMCPU.
1173	*
1174	* @remarks Called with interrupts disabled.
1175	*/
1176	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1177	{
1178	NOREF(pVM);
1179	AssertPtr(pVCpu);
1180	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1181	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1182
1183	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1184	AssertPtr(pCpu);
1185
1186	pVCpu->hm.s.TlbShootdown.cPages = 0;
1187	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1188	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1189	pVCpu->hm.s.fForceTLBFlush = false;
1190	return;
1191	}
1192
1193
1194	/**
1195	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1196	*
1197	* @param pVM Pointer to the VM.
1198	* @param pVCpu Pointer to the VMCPU.
1199	* @remarks All references to "ASID" in this function pertains to "VPID" in
1200	* Intel's nomenclature. The reason is, to avoid confusion in compare
1201	* statements since the host-CPU copies are named "ASID".
1202	*
1203	* @remarks Called with interrupts disabled.
1204	*/
1205	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1206	{
1207	AssertPtr(pVM);
1208	AssertPtr(pVCpu);
1209	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1210	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1211	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1212
1213	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1214	AssertPtr(pCpu);
1215
1216	/*
1217	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1218	* This can happen both for start & resume due to long jumps back to ring-3.
1219	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1220	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1221	*/
1222	bool fNewASID = false;
1223	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1224	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1225	{
1226	pVCpu->hm.s.fForceTLBFlush = true;
1227	fNewASID = true;
1228	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1229	}
1230
1231	/*
1232	* Check for explicit TLB shootdowns.
1233	*/
1234	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1235	{
1236	pVCpu->hm.s.fForceTLBFlush = true;
1237	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1238	}
1239
1240	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1241	if (pVCpu->hm.s.fForceTLBFlush)
1242	{
1243	if (fNewASID)
1244	{
1245	++pCpu->uCurrentAsid;
1246	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1247	{
1248	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1249	pCpu->cTlbFlushes++;
1250	pCpu->fFlushAsidBeforeUse = true;
1251	}
1252
1253	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1254	if (pCpu->fFlushAsidBeforeUse)
1255	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1256	}
1257	else
1258	{
1259	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1260	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1261	else
1262	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1263	}
1264
1265	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1266	pVCpu->hm.s.fForceTLBFlush = false;
1267	}
1268	else
1269	{
1270	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1271	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1272	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1273	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1274
1275	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1276	* not be executed. See hmQueueInvlPage() where it is commented
1277	* out. Support individual entry flushing someday. */
1278	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1279	{
1280	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1281
1282	/*
1283	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1284	* as supported by the CPU.
1285	*/
1286	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1287	{
1288	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1289	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1290	}
1291	else
1292	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1293	}
1294	else
1295	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1296	}
1297	pVCpu->hm.s.TlbShootdown.cPages = 0;
1298	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1299
1300	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1301	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1302	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1304	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1305	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1306
1307	/* Update VMCS with the VPID. */
1308	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1309	AssertRC(rc);
1310	}
1311
1312
1313	/**
1314	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1315	*
1316	* @returns VBox status code.
1317	* @param pVM Pointer to the VM.
1318	* @param pVCpu Pointer to the VMCPU.
1319	*
1320	* @remarks Called with interrupts disabled.
1321	*/
1322	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1323	{
1324	AssertPtr(pVM);
1325	AssertPtr(pVCpu);
1326	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1327	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1328
1329	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1330	AssertPtr(pCpu);
1331
1332	/*
1333	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1334	* This can happen both for start & resume due to long jumps back to ring-3.
1335	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1336	*/
1337	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1338	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1339	{
1340	pVCpu->hm.s.fForceTLBFlush = true;
1341	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1342	}
1343
1344	/* Check for explicit TLB shootdown flushes. */
1345	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1346	{
1347	pVCpu->hm.s.fForceTLBFlush = true;
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	}
1350
1351	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1352	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1353
1354	if (pVCpu->hm.s.fForceTLBFlush)
1355	{
1356	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1357	pVCpu->hm.s.fForceTLBFlush = false;
1358	}
1359	else
1360	{
1361	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1362	* not be executed. See hmQueueInvlPage() where it is commented
1363	* out. Support individual entry flushing someday. */
1364	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1365	{
1366	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1368	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1369	}
1370	else
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1372	}
1373
1374	pVCpu->hm.s.TlbShootdown.cPages = 0;
1375	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1376	}
1377
1378
1379	/**
1380	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1381	*
1382	* @returns VBox status code.
1383	* @param pVM Pointer to the VM.
1384	* @param pVCpu Pointer to the VMCPU.
1385	*
1386	* @remarks Called with interrupts disabled.
1387	*/
1388	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1389	{
1390	AssertPtr(pVM);
1391	AssertPtr(pVCpu);
1392	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1393	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1394
1395	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1396
1397	/*
1398	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1399	* This can happen both for start & resume due to long jumps back to ring-3.
1400	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1401	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1402	*/
1403	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1404	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1405	{
1406	pVCpu->hm.s.fForceTLBFlush = true;
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1408	}
1409
1410	/* Check for explicit TLB shootdown flushes. */
1411	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1412	{
1413	/*
1414	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1415	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1416	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1417	*/
1418	pVCpu->hm.s.fForceTLBFlush = true;
1419	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1420	}
1421
1422	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1423	if (pVCpu->hm.s.fForceTLBFlush)
1424	{
1425	++pCpu->uCurrentAsid;
1426	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1427	{
1428	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1429	pCpu->fFlushAsidBeforeUse = true;
1430	pCpu->cTlbFlushes++;
1431	}
1432
1433	pVCpu->hm.s.fForceTLBFlush = false;
1434	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1435	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1436	if (pCpu->fFlushAsidBeforeUse)
1437	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1438	}
1439	else
1440	{
1441	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1442	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1443	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1444	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1445
1446	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1447	* not be executed. See hmQueueInvlPage() where it is commented
1448	* out. Support individual entry flushing someday. */
1449	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1450	{
1451	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1452	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1453	{
1454	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1455	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1456	}
1457	else
1458	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1459	}
1460	else
1461	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1462	}
1463
1464	pVCpu->hm.s.TlbShootdown.cPages = 0;
1465	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1466
1467	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1468	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1469	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1471	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1472	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1473
1474	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1475	AssertRC(rc);
1476	}
1477
1478
1479	/**
1480	* Flushes the guest TLB entry based on CPU capabilities.
1481	*
1482	* @param pVCpu Pointer to the VMCPU.
1483	*/
1484	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1485	{
1486	PVM pVM = pVCpu->CTX_SUFF(pVM);
1487	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1488	{
1489	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1491	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1492	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1493	default:
1494	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1495	break;
1496	}
1497	}
1498
1499
1500	/**
1501	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1502	* TLB entries from the host TLB before VM-entry.
1503	*
1504	* @returns VBox status code.
1505	* @param pVM Pointer to the VM.
1506	*/
1507	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1508	{
1509	/*
1510	* Determine optimal flush type for nested paging.
1511	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1512	* guest execution (see hmR3InitFinalizeR0()).
1513	*/
1514	if (pVM->hm.s.fNestedPaging)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1517	{
1518	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1520	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1521	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1522	else
1523	{
1524	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1525	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1526	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1527	}
1528
1529	/* Make sure the write-back cacheable memory type for EPT is supported. */
1530	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1531	{
1532	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1533	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1534	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1535	}
1536	}
1537	else
1538	{
1539	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1540	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1541	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1542	}
1543	}
1544
1545	/*
1546	* Determine optimal flush type for VPID.
1547	*/
1548	if (pVM->hm.s.vmx.fVpid)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1551	{
1552	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1554	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1555	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1556	else
1557	{
1558	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1561	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1562	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1563	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1564	pVM->hm.s.vmx.fVpid = false;
1565	}
1566	}
1567	else
1568	{
1569	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1570	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1571	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1572	pVM->hm.s.vmx.fVpid = false;
1573	}
1574	}
1575
1576	/*
1577	* Setup the handler for flushing tagged-TLBs.
1578	*/
1579	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1581	else if (pVM->hm.s.fNestedPaging)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1583	else if (pVM->hm.s.vmx.fVpid)
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1585	else
1586	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1587	return VINF_SUCCESS;
1588	}
1589
1590
1591	/**
1592	* Sets up pin-based VM-execution controls in the VMCS.
1593	*
1594	* @returns VBox status code.
1595	* @param pVM Pointer to the VM.
1596	* @param pVCpu Pointer to the VMCPU.
1597	*/
1598	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1599	{
1600	AssertPtr(pVM);
1601	AssertPtr(pVCpu);
1602
1603	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1604	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1605
1606	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1607	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1608	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1609
1610	/* Enable the VMX preemption timer. */
1611	if (pVM->hm.s.vmx.fUsePreemptTimer)
1612	{
1613	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1614	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1615	}
1616
1617	if ((val & zap) != val)
1618	{
1619	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1620	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1621	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1622	}
1623
1624	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1625	AssertRCReturn(rc, rc);
1626
1627	/* Update VCPU with the currently set pin-based VM-execution controls. */
1628	pVCpu->hm.s.vmx.u32PinCtls = val;
1629	return rc;
1630	}
1631
1632
1633	/**
1634	* Sets up processor-based VM-execution controls in the VMCS.
1635	*
1636	* @returns VBox status code.
1637	* @param pVM Pointer to the VM.
1638	* @param pVMCPU Pointer to the VMCPU.
1639	*/
1640	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1641	{
1642	AssertPtr(pVM);
1643	AssertPtr(pVCpu);
1644
1645	int rc = VERR_INTERNAL_ERROR_5;
1646	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1647	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1648
1649	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1654	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1655	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1656
1657	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1658	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1659	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1660	{
1661	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1662	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1663	}
1664
1665	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1666	if (!pVM->hm.s.fNestedPaging)
1667	{
1668	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1669	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1670	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1671	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1672	}
1673
1674	/* Use TPR shadowing if supported by the CPU. */
1675	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1676	{
1677	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1678	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1679	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1680	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1681	AssertRCReturn(rc, rc);
1682
1683	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1684	/* CR8 writes causes a VM-exit based on TPR threshold. */
1685	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1686	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1687	}
1688	else
1689	{
1690	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1691	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1692	}
1693
1694	/* Use MSR-bitmaps if supported by the CPU. */
1695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1696	{
1697	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1698
1699	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1701	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1702	AssertRCReturn(rc, rc);
1703
1704	/*
1705	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1706	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1707	*/
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1716	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1717	}
1718
1719	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1720	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1721	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1722
1723	if ((val & zap) != val)
1724	{
1725	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1726	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1727	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1728	}
1729
1730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1731	AssertRCReturn(rc, rc);
1732
1733	/* Update VCPU with the currently set processor-based VM-execution controls. */
1734	pVCpu->hm.s.vmx.u32ProcCtls = val;
1735
1736	/*
1737	* Secondary processor-based VM-execution controls.
1738	*/
1739	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1740	{
1741	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1742	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1743
1744	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1745	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1746
1747	if (pVM->hm.s.fNestedPaging)
1748	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1749	else
1750	{
1751	/*
1752	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1753	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1754	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1755	*/
1756	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1757	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1758	}
1759
1760	if (pVM->hm.s.vmx.fVpid)
1761	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1762
1763	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1764	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1765
1766	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1767	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1768	* done dynamically. */
1769	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1770	{
1771	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1772	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1773	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1774	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1775	AssertRCReturn(rc, rc);
1776	}
1777
1778	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1779	{
1780	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1781	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1782	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1783	}
1784
1785	if ((val & zap) != val)
1786	{
1787	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1788	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1789	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1790	}
1791
1792	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1793	AssertRCReturn(rc, rc);
1794
1795	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1796	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1797	}
1798
1799	return VINF_SUCCESS;
1800	}
1801
1802
1803	/**
1804	* Sets up miscellaneous (everything other than Pin & Processor-based
1805	* VM-execution) control fields in the VMCS.
1806	*
1807	* @returns VBox status code.
1808	* @param pVM Pointer to the VM.
1809	* @param pVCpu Pointer to the VMCPU.
1810	*/
1811	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1812	{
1813	AssertPtr(pVM);
1814	AssertPtr(pVCpu);
1815
1816	/** @todo Shouldn't we able to avoid initializing with 0? */
1817	int rc = VERR_GENERAL_FAILURE;
1818
1819	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1820	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1821	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1822
1823	/*
1824	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1825	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1826	* We thus use the exception bitmap to control it rather than use both.
1827	*/
1828	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1829	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1830
1831	/** @todo Explore possibility of using IO-bitmaps. */
1832	/* All IO & IOIO instructions cause VM-exits. */
1833	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1834	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1835
1836	/* Initialize the MSR-bitmap area. */
1837	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1838	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1839	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1840
1841	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1842	/* Setup MSR autoloading/storing. */
1843	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1845	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1846	AssertRCReturn(rc, rc);
1847	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1848	AssertRCReturn(rc, rc);
1849
1850	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1851	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1852	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1853	AssertRCReturn(rc, rc);
1854	#endif
1855
1856	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1857	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1858	AssertRCReturn(rc, rc);
1859
1860	/* Setup debug controls */
1861	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1862	AssertRCReturn(rc, rc);
1863	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1864	AssertRCReturn(rc, rc);
1865	return rc;
1866	}
1867
1868
1869	/**
1870	* Sets up the initial exception bitmap in the VMCS based on static conditions
1871	* (i.e. conditions that cannot ever change at runtime).
1872	*
1873	* @returns VBox status code.
1874	* @param pVM Pointer to the VM.
1875	* @param pVCpu Pointer to the VMCPU.
1876	*/
1877	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1878	{
1879	AssertPtr(pVM);
1880	AssertPtr(pVCpu);
1881
1882	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1883
1884	uint32_t u32XcptBitmap = 0;
1885
1886	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1887	if (!pVM->hm.s.fNestedPaging)
1888	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1889
1890	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1891	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1892	AssertRCReturn(rc, rc);
1893	return rc;
1894	}
1895
1896
1897	/**
1898	* Sets up the initial guest-state mask. The guest-state mask is consulted
1899	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1900	* for the nested virtualization case (as it would cause a VM-exit).
1901	*
1902	* @param pVCpu Pointer to the VMCPU.
1903	*/
1904	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1905	{
1906	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1907	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1908	return VINF_SUCCESS;
1909	}
1910
1911
1912	/**
1913	* Does per-VM VT-x initialization.
1914	*
1915	* @returns VBox status code.
1916	* @param pVM Pointer to the VM.
1917	*/
1918	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1919	{
1920	LogFlowFunc(("pVM=%p\n", pVM));
1921
1922	int rc = hmR0VmxStructsAlloc(pVM);
1923	if (RT_FAILURE(rc))
1924	{
1925	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1926	return rc;
1927	}
1928
1929	return VINF_SUCCESS;
1930	}
1931
1932
1933	/**
1934	* Does per-VM VT-x termination.
1935	*
1936	* @returns VBox status code.
1937	* @param pVM Pointer to the VM.
1938	*/
1939	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1940	{
1941	LogFlowFunc(("pVM=%p\n", pVM));
1942
1943	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1944	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1945	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1946	#endif
1947	hmR0VmxStructsFree(pVM);
1948	return VINF_SUCCESS;
1949	}
1950
1951
1952	/**
1953	* Sets up the VM for execution under VT-x.
1954	* This function is only called once per-VM during initalization.
1955	*
1956	* @returns VBox status code.
1957	* @param pVM Pointer to the VM.
1958	*/
1959	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1960	{
1961	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1962	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1963
1964	LogFlowFunc(("pVM=%p\n", pVM));
1965
1966	/*
1967	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1968	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1969	*/
1970	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1971	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1972	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1973	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1974	{
1975	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1976	return VERR_INTERNAL_ERROR;
1977	}
1978
1979	/* Initialize these always, see hmR3InitFinalizeR0().*/
1980	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1981	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1982
1983	/* Setup the tagged-TLB flush handlers. */
1984	int rc = hmR0VmxSetupTaggedTlb(pVM);
1985	if (RT_FAILURE(rc))
1986	{
1987	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1988	return rc;
1989	}
1990
1991	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1992	{
1993	PVMCPU pVCpu = &pVM->aCpus[i];
1994	AssertPtr(pVCpu);
1995	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1996
1997	/* Set revision dword at the beginning of the VMCS structure. */
1998	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1999
2000	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2001	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2002	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2003	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2004
2005	/* Load this VMCS as the current VMCS. */
2006	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2007	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2008	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2009
2010	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2011	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2012	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2013
2014	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2015	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2016	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2017
2018	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2019	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2020	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2021
2022	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2023	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2024	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2025
2026	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2027	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2028	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2029
2030	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2031	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2032	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2033	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2034	#endif
2035
2036	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2037	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2038	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2040
2041	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2042	}
2043
2044	return VINF_SUCCESS;
2045	}
2046
2047
2048	/**
2049	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2050	* the VMCS.
2051	*
2052	* @returns VBox status code.
2053	* @param pVM Pointer to the VM.
2054	* @param pVCpu Pointer to the VMCPU.
2055	*/
2056	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2057	{
2058	RTCCUINTREG uReg = ASMGetCR0();
2059	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2060	AssertRCReturn(rc, rc);
2061
2062	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2063	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2064	if (HMVMX_IS_64BIT_HOST_MODE())
2065	{
2066	uint64_t uRegCR3 = hmR0Get64bitCR3();
2067	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2068	}
2069	else
2070	#endif
2071	{
2072	uReg = ASMGetCR3();
2073	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2074	}
2075	AssertRCReturn(rc, rc);
2076
2077	uReg = ASMGetCR4();
2078	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2079	AssertRCReturn(rc, rc);
2080	return rc;
2081	}
2082
2083
2084	/**
2085	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2086	* the host-state area in the VMCS.
2087	*
2088	* @returns VBox status code.
2089	* @param pVM Pointer to the VM.
2090	* @param pVCpu Pointer to the VMCPU.
2091	*/
2092	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2093	{
2094	int rc = VERR_INTERNAL_ERROR_5;
2095	RTSEL uSelCS = 0;
2096	RTSEL uSelSS = 0;
2097	RTSEL uSelDS = 0;
2098	RTSEL uSelES = 0;
2099	RTSEL uSelFS = 0;
2100	RTSEL uSelGS = 0;
2101	RTSEL uSelTR = 0;
2102
2103	/*
2104	* Host Selector registers.
2105	*/
2106	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2107	if (HMVMX_IS_64BIT_HOST_MODE())
2108	{
2109	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2110	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2111	}
2112	else
2113	{
2114	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2115	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2116	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2117	}
2118	#else
2119	uSelCS = ASMGetCS();
2120	uSelSS = ASMGetSS();
2121	#endif
2122
2123	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2124	uSelTR = ASMGetTR();
2125
2126	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2127	/** @todo Verify if we have any platform that actually run with DS or ES with
2128	* RPL != 0 in kernel space. */
2129	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2130	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2131	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2132	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2133	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2134	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2135	Assert(uSelCS != 0);
2136	Assert(uSelTR != 0);
2137
2138	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2139	#if 0
2140	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2141	Assert(uSelSS != 0);
2142	#endif
2143
2144	/* Write these host selector fields into the host-state area in the VMCS. */
2145	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2146	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2147	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2148	#if 0
2149	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2150	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2151	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2152	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2153	#endif
2154	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2155
2156	/*
2157	* Host GDTR and IDTR.
2158	*/
2159	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2160	* be safe to -not- save and restore GDTR and IDTR in the assembly
2161	* code and just do it here and don't care if the limits are zapped on
2162	* VM-exit. */
2163	RTGDTR Gdtr;
2164	RT_ZERO(Gdtr);
2165	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2166	if (HMVMX_IS_64BIT_HOST_MODE())
2167	{
2168	X86XDTR64 Gdtr64;
2169	X86XDTR64 Idtr64;
2170	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2171	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2172	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2173
2174	Gdtr.cbGdt = Gdtr64.cb;
2175	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2176	}
2177	else
2178	#endif
2179	{
2180	RTIDTR Idtr;
2181	ASMGetGDTR(&Gdtr);
2182	ASMGetIDTR(&Idtr);
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2184	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2185	}
2186
2187	/*
2188	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2189	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2190	*/
2191	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2192	{
2193	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2194	return VERR_VMX_INVALID_HOST_STATE;
2195	}
2196
2197	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2198	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2199	if (HMVMX_IS_64BIT_HOST_MODE())
2200	{
2201	/* We need the 64-bit TR base for hybrid darwin. */
2202	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2203	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2204	}
2205	else
2206	#endif
2207	{
2208	uintptr_t uTRBase;
2209	#if HC_ARCH_BITS == 64
2210	uTRBase = X86DESC64_BASE(pDesc);
2211	#else
2212	uTRBase = X86DESC_BASE(pDesc);
2213	#endif
2214	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2215	}
2216	AssertRCReturn(rc, rc);
2217
2218	/*
2219	* Host FS base and GS base.
2220	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2221	* would take care of the bases. In 64-bit, the MSRs come into play.
2222	*/
2223	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2224	if (HMVMX_IS_64BIT_HOST_MODE())
2225	{
2226	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2227	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2228	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2229	AssertRCReturn(rc, rc);
2230	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2231	AssertRCReturn(rc, rc);
2232	}
2233	#endif
2234	return rc;
2235	}
2236
2237
2238	/**
2239	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2240	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2241	* the host after every successful VM exit.
2242	*
2243	* @returns VBox status code.
2244	* @param pVM Pointer to the VM.
2245	* @param pVCpu Pointer to the VMCPU.
2246	*/
2247	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2248	{
2249	AssertPtr(pVCpu);
2250	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2251
2252	int rc = VINF_SUCCESS;
2253	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2254	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2255	uint32_t cHostMsrs = 0;
2256	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2257
2258	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2259	{
2260	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2261	pHostMsr->u32Reserved = 0;
2262	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2263	if (CPUMIsGuestInLongMode(pVCpu))
2264	{
2265	/* Must match the EFER value in our 64 bits switcher. */
2266	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2267	}
2268	else
2269	# endif
2270	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2271	pHostMsr++; cHostMsrs++;
2272	}
2273
2274	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2275	if (HMVMX_IS_64BIT_HOST_MODE())
2276	{
2277	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2278	pHostMsr->u32Reserved = 0;
2279	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2280	pHostMsr++; cHostMsrs++;
2281	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2282	pHostMsr->u32Reserved = 0;
2283	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2284	pHostMsr++; cHostMsrs++;
2285	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2286	pHostMsr->u32Reserved = 0;
2287	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2288	pHostMsr++; cHostMsrs++;
2289	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2290	pHostMsr->u32Reserved = 0;
2291	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2292	pHostMsr++; cHostMsrs++;
2293	}
2294	# endif
2295
2296	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2297	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2298	{
2299	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2300	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2301	}
2302
2303	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2304	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2305
2306	/*
2307	* Host Sysenter MSRs.
2308	*/
2309	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2310	AssertRCReturn(rc, rc);
2311	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2312	if (HMVMX_IS_64BIT_HOST_MODE())
2313	{
2314	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2315	AssertRCReturn(rc, rc);
2316	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2317	}
2318	else
2319	{
2320	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2321	AssertRCReturn(rc, rc);
2322	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2323	}
2324	# elif HC_ARCH_BITS == 32
2325	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2326	AssertRCReturn(rc, rc);
2327	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2328	# else
2329	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2330	AssertRCReturn(rc, rc);
2331	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2332	# endif
2333	AssertRCReturn(rc, rc);
2334
2335	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2336	* hmR0VmxSetupExitCtls() !! */
2337	return rc;
2338	}
2339
2340
2341	/**
2342	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2343	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2344	* controls".
2345	*
2346	* @returns VBox status code.
2347	* @param pVCpu Pointer to the VMCPU.
2348	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2349	* out-of-sync. Make sure to update the required fields
2350	* before using them.
2351	*
2352	* @remarks No-long-jump zone!!!
2353	*/
2354	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2355	{
2356	int rc = VINF_SUCCESS;
2357	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2358	{
2359	PVM pVM = pVCpu->CTX_SUFF(pVM);
2360	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2361	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2362
2363	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2364	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2365
2366	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2367	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2368	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2369	else
2370	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2371
2372	/*
2373	* The following should not be set (since we're not in SMM mode):
2374	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2375	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2376	*/
2377
2378	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2379	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2380	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2381
2382	if ((val & zap) != val)
2383	{
2384	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2385	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2386	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2387	}
2388
2389	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2390	AssertRCReturn(rc, rc);
2391
2392	/* Update VCPU with the currently set VM-exit controls. */
2393	pVCpu->hm.s.vmx.u32EntryCtls = val;
2394	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2395	}
2396	return rc;
2397	}
2398
2399
2400	/**
2401	* Sets up the VM-exit controls in the VMCS.
2402	*
2403	* @returns VBox status code.
2404	* @param pVM Pointer to the VM.
2405	* @param pVCpu Pointer to the VMCPU.
2406	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2407	* out-of-sync. Make sure to update the required fields
2408	* before using them.
2409	*
2410	* @remarks requires EFER.
2411	*/
2412	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2413	{
2414	int rc = VINF_SUCCESS;
2415	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2416	{
2417	PVM pVM = pVCpu->CTX_SUFF(pVM);
2418	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2419	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2420
2421	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2422	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2423
2424	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2425	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2426	if (HMVMX_IS_64BIT_HOST_MODE())
2427	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2428	else
2429	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2430	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2431	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2432	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2433	else
2434	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2435	#endif
2436
2437	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2438	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2439
2440	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2441	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2442	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2443	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2444	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2445
2446	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2447	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2448
2449	if ((val & zap) != val)
2450	{
2451	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2452	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2453	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2454	}
2455
2456	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2457	AssertRCReturn(rc, rc);
2458
2459	/* Update VCPU with the currently set VM-exit controls. */
2460	pVCpu->hm.s.vmx.u32ExitCtls = val;
2461	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2462	}
2463	return rc;
2464	}
2465
2466
2467	/**
2468	* Loads the guest APIC and related state.
2469	*
2470	* @returns VBox status code.
2471	* @param pVM Pointer to the VM.
2472	* @param pVCpu Pointer to the VMCPU.
2473	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2474	* out-of-sync. Make sure to update the required fields
2475	* before using them.
2476	*/
2477	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2478	{
2479	int rc = VINF_SUCCESS;
2480	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2481	{
2482	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2483	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2484	{
2485	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2486
2487	bool fPendingIntr = false;
2488	uint8_t u8GuestTpr = 0;
2489	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2490	AssertRCReturn(rc, rc);
2491
2492	/*
2493	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2494	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2495	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2496	* the interrupt when we VM-exit for other reasons.
2497	*/
2498	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2499	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2500	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2501	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2502
2503	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2504	AssertRCReturn(rc, rc);
2505
2506	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2507	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2508	{
2509	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2510	pMixedCtx->msrLSTAR = u8GuestTpr;
2511	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2512	{
2513	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2514	if (fPendingIntr)
2515	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2516	else
2517	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518	}
2519	}
2520	}
2521
2522	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2523	}
2524	return rc;
2525	}
2526
2527
2528	/**
2529	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2530	*
2531	* @returns
2532	* @param pVCpu Pointer to the VMCPU.
2533	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2534	* out-of-sync. Make sure to update the required fields
2535	* before using them.
2536	*
2537	* @remarks No-long-jump zone!!!
2538	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2539	*/
2540	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2541	{
2542	/*
2543	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2544	* inhibit interrupts or clear any existing interrupt-inhibition.
2545	*/
2546	uint32_t uIntrState = 0;
2547	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2548	{
2549	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2550	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2551	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2552	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2553	{
2554	/*
2555	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2556	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2557	*/
2558	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2559	}
2560	else if (pMixedCtx->eflags.Bits.u1IF)
2561	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2562	else
2563	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2564	}
2565	return uIntrState;
2566	}
2567
2568
2569	/**
2570	* Loads the guest's interruptibility-state into the guest-state area in the
2571	* VMCS.
2572	*
2573	* @returns VBox status code.
2574	* @param pVCpu Pointer to the VMCPU.
2575	* @param uIntrState The interruptibility-state to set.
2576	*/
2577	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2578	{
2579	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2580	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2581	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2582	AssertRCReturn(rc, rc);
2583	return rc;
2584	}
2585
2586
2587	/**
2588	* Loads the guest's RIP into the guest-state area in the VMCS.
2589	*
2590	* @returns VBox status code.
2591	* @param pVCpu Pointer to the VMCPU.
2592	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2593	* out-of-sync. Make sure to update the required fields
2594	* before using them.
2595	*
2596	* @remarks No-long-jump zone!!!
2597	*/
2598	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2599	{
2600	int rc = VINF_SUCCESS;
2601	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2602	{
2603	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2604	AssertRCReturn(rc, rc);
2605	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2606	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2607	}
2608	return rc;
2609	}
2610
2611
2612	/**
2613	* Loads the guest's RSP into the guest-state area in the VMCS.
2614	*
2615	* @returns VBox status code.
2616	* @param pVCpu Pointer to the VMCPU.
2617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2618	* out-of-sync. Make sure to update the required fields
2619	* before using them.
2620	*
2621	* @remarks No-long-jump zone!!!
2622	*/
2623	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2624	{
2625	int rc = VINF_SUCCESS;
2626	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2627	{
2628	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2629	AssertRCReturn(rc, rc);
2630	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2631	}
2632	return rc;
2633	}
2634
2635
2636	/**
2637	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2638	*
2639	* @returns VBox status code.
2640	* @param pVCpu Pointer to the VMCPU.
2641	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2642	* out-of-sync. Make sure to update the required fields
2643	* before using them.
2644	*
2645	* @remarks No-long-jump zone!!!
2646	*/
2647	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2648	{
2649	int rc = VINF_SUCCESS;
2650	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2651	{
2652	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2653	Let us assert it as such and use 32-bit VMWRITE. */
2654	Assert(!(pMixedCtx->rflags.u64 >> 32));
2655	X86EFLAGS uEFlags = pMixedCtx->eflags;
2656	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2657	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2658
2659	/*
2660	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2661	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2662	*/
2663	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2664	{
2665	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2666	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2667	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2668	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2669	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2670	}
2671
2672	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2673	AssertRCReturn(rc, rc);
2674
2675	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2676	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2677	}
2678	return rc;
2679	}
2680
2681
2682	/**
2683	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2684	*
2685	* @returns VBox status code.
2686	* @param pVCpu Pointer to the VMCPU.
2687	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2688	* out-of-sync. Make sure to update the required fields
2689	* before using them.
2690	*
2691	* @remarks No-long-jump zone!!!
2692	*/
2693	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2694	{
2695	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2696	AssertRCReturn(rc, rc);
2697	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2698	AssertRCReturn(rc, rc);
2699	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2700	AssertRCReturn(rc, rc);
2701	return rc;
2702	}
2703
2704
2705	/**
2706	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2707	* in the VMCS.
2708	*
2709	* @returns VBox status code.
2710	* @param pVM Pointer to the VM.
2711	* @param pVCpu Pointer to the VMCPU.
2712	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2713	* out-of-sync. Make sure to update the required fields
2714	* before using them.
2715	*
2716	* @remarks No-long-jump zone!!!
2717	*/
2718	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2719	{
2720	int rc = VINF_SUCCESS;
2721	PVM pVM = pVCpu->CTX_SUFF(pVM);
2722
2723	/*
2724	* Guest CR0.
2725	* Guest FPU.
2726	*/
2727	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2728	{
2729	Assert(!(pCtx->cr0 >> 32));
2730	uint32_t u32GuestCR0 = pCtx->cr0;
2731
2732	/* The guest's view (read access) of its CR0 is unblemished. */
2733	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2734	AssertRCReturn(rc, rc);
2735	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2736
2737	/* Setup VT-x's view of the guest CR0. */
2738	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2739	if (pVM->hm.s.fNestedPaging)
2740	{
2741	if (CPUMIsGuestPagingEnabledEx(pCtx))
2742	{
2743	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2744	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2745	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2746	}
2747	else
2748	{
2749	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2750	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2751	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2752	}
2753
2754	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2755	AssertRCReturn(rc, rc);
2756	}
2757	else
2758	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2759
2760	/*
2761	* Guest FPU bits.
2762	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2763	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2764	*/
2765	u32GuestCR0 \|= X86_CR0_NE;
2766	bool fInterceptNM = false;
2767	if (CPUMIsGuestFPUStateActive(pVCpu))
2768	{
2769	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2770	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2771	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2772	}
2773	else
2774	{
2775	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2776	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2777	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2778	}
2779
2780	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2781	bool fInterceptMF = false;
2782	if (!(pCtx->cr0 & X86_CR0_NE))
2783	fInterceptMF = true;
2784
2785	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2786	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2787	{
2788	Assert(PDMVmmDevHeapIsEnabled(pVM));
2789	Assert(pVM->hm.s.vmx.pRealModeTSS);
2790	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2791	fInterceptNM = true;
2792	fInterceptMF = true;
2793	}
2794	else
2795	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2796
2797	if (fInterceptNM)
2798	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2799	else
2800	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2801
2802	if (fInterceptMF)
2803	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2804	else
2805	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2806
2807	/* Additional intercepts for debugging, define these yourself explicitly. */
2808	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2809	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2810	\| RT_BIT(X86_XCPT_DB)
2811	\| RT_BIT(X86_XCPT_DE)
2812	\| RT_BIT(X86_XCPT_NM)
2813	\| RT_BIT(X86_XCPT_UD)
2814	\| RT_BIT(X86_XCPT_NP)
2815	\| RT_BIT(X86_XCPT_SS)
2816	\| RT_BIT(X86_XCPT_GP)
2817	\| RT_BIT(X86_XCPT_PF)
2818	\| RT_BIT(X86_XCPT_MF);
2819	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2820	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2821	#endif
2822
2823	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2824
2825	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2826	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2827	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2828	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2829	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2830	else
2831	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2832
2833	u32GuestCR0 \|= uSetCR0;
2834	u32GuestCR0 &= uZapCR0;
2835	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2836
2837	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2838	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2839	AssertRCReturn(rc, rc);
2840	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2841	AssertRCReturn(rc, rc);
2842	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2843
2844	/*
2845	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2846	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2847	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2848	*/
2849	uint32_t u32CR0Mask = 0;
2850	u32CR0Mask = X86_CR0_PE
2851	\| X86_CR0_NE
2852	\| X86_CR0_WP
2853	\| X86_CR0_PG
2854	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2855	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2856	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2857	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2858	u32CR0Mask &= ~X86_CR0_PE;
2859	if (pVM->hm.s.fNestedPaging)
2860	u32CR0Mask &= ~X86_CR0_WP;
2861
2862	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2863	if (fInterceptNM)
2864	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2865	else
2866	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2867
2868	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2869	pVCpu->hm.s.vmx.cr0_mask = u32CR0Mask;
2870	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2871	AssertRCReturn(rc, rc);
2872
2873	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2874	}
2875
2876	/*
2877	* Guest CR2.
2878	* It's always loaded in the assembler code. Nothing to do here.
2879	*/
2880
2881	/*
2882	* Guest CR3.
2883	*/
2884	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2885	{
2886	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2887	if (pVM->hm.s.fNestedPaging)
2888	{
2889	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2890
2891	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2892	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2893	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2894	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2895
2896	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2897	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2898	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2899
2900	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2901	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2902	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2903	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2904
2905	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2906	AssertRCReturn(rc, rc);
2907	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2908
2909	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2910	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2911	{
2912	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2913	if (CPUMIsGuestInPAEModeEx(pCtx))
2914	{
2915	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
2916	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
2917	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
2918	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
2919	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
2920	AssertRCReturn(rc, rc);
2921	}
2922
2923	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2924	have Unrestricted Execution to handle the guest when it's not using paging. */
2925	GCPhysGuestCR3 = pCtx->cr3;
2926	}
2927	else
2928	{
2929	/*
2930	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2931	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2932	* EPT takes care of translating it to host-physical addresses.
2933	*/
2934	RTGCPHYS GCPhys;
2935	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2936	Assert(PDMVmmDevHeapIsEnabled(pVM));
2937
2938	/* We obtain it here every time as the guest could have relocated this PCI region. */
2939	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2940	AssertRCReturn(rc, rc);
2941
2942	GCPhysGuestCR3 = GCPhys;
2943	}
2944
2945	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
2946	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2947	}
2948	else
2949	{
2950	/* Non-nested paging case, just use the hypervisor's CR3. */
2951	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2952
2953	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
2954	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
2955	}
2956	AssertRCReturn(rc, rc);
2957
2958	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2959	}
2960
2961	/*
2962	* Guest CR4.
2963	*/
2964	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2965	{
2966	Assert(!(pCtx->cr4 >> 32));
2967	uint32_t u32GuestCR4 = pCtx->cr4;
2968
2969	/* The guest's view of its CR4 is unblemished. */
2970	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2971	AssertRCReturn(rc, rc);
2972	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2973
2974	/* Setup VT-x's view of the guest CR4. */
2975	/*
2976	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2977	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2978	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2979	*/
2980	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2981	{
2982	Assert(pVM->hm.s.vmx.pRealModeTSS);
2983	Assert(PDMVmmDevHeapIsEnabled(pVM));
2984	u32GuestCR4 &= ~X86_CR4_VME;
2985	}
2986
2987	if (pVM->hm.s.fNestedPaging)
2988	{
2989	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2990	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2991	{
2992	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2993	u32GuestCR4 \|= X86_CR4_PSE;
2994	/* Our identity mapping is a 32 bits page directory. */
2995	u32GuestCR4 &= ~X86_CR4_PAE;
2996	}
2997	/* else use guest CR4.*/
2998	}
2999	else
3000	{
3001	/*
3002	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3003	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3004	*/
3005	switch (pVCpu->hm.s.enmShadowMode)
3006	{
3007	case PGMMODE_REAL: /* Real-mode. */
3008	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3009	case PGMMODE_32_BIT: /* 32-bit paging. */
3010	{
3011	u32GuestCR4 &= ~X86_CR4_PAE;
3012	break;
3013	}
3014
3015	case PGMMODE_PAE: /* PAE paging. */
3016	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3017	{
3018	u32GuestCR4 \|= X86_CR4_PAE;
3019	break;
3020	}
3021
3022	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3023	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3024	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3025	break;
3026	#endif
3027	default:
3028	AssertFailed();
3029	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3030	}
3031	}
3032
3033	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3034	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3035	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3036	u32GuestCR4 \|= uSetCR4;
3037	u32GuestCR4 &= uZapCR4;
3038
3039	/* Write VT-x's view of the guest CR4 into the VMCS. */
3040	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3041	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3042	AssertRCReturn(rc, rc);
3043
3044	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3045	uint32_t u32CR4Mask = 0;
3046	u32CR4Mask = X86_CR4_VME
3047	\| X86_CR4_PAE
3048	\| X86_CR4_PGE
3049	\| X86_CR4_PSE
3050	\| X86_CR4_VMXE;
3051	pVCpu->hm.s.vmx.cr4_mask = u32CR4Mask;
3052	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3053	AssertRCReturn(rc, rc);
3054
3055	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3056	}
3057	return rc;
3058	}
3059
3060
3061	/**
3062	* Loads the guest debug registers into the guest-state area in the VMCS.
3063	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3064	*
3065	* @returns VBox status code.
3066	* @param pVCpu Pointer to the VMCPU.
3067	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3068	* out-of-sync. Make sure to update the required fields
3069	* before using them.
3070	*
3071	* @remarks No-long-jump zone!!!
3072	*/
3073	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3074	{
3075	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3076	return VINF_SUCCESS;
3077
3078	#ifdef VBOX_STRICT
3079	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3080	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3081	{
3082	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3083	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3084	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3085	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3086	}
3087	#endif
3088
3089	int rc = VERR_INTERNAL_ERROR_5;
3090	PVM pVM = pVCpu->CTX_SUFF(pVM);
3091	bool fInterceptDB = false;
3092	bool fInterceptMovDRx = false;
3093	if (DBGFIsStepping(pVCpu))
3094	{
3095	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3096	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3097	{
3098	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3099	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3100	AssertRCReturn(rc, rc);
3101	Assert(fInterceptDB == false);
3102	}
3103	else
3104	fInterceptDB = true;
3105	}
3106
3107	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3108	{
3109	if (!CPUMIsHyperDebugStateActive(pVCpu))
3110	{
3111	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3112	AssertRC(rc);
3113	}
3114	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3115	fInterceptMovDRx = true;
3116	}
3117	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3118	{
3119	if (!CPUMIsGuestDebugStateActive(pVCpu))
3120	{
3121	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3122	AssertRC(rc);
3123	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3124	}
3125	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3126	Assert(fInterceptMovDRx == false);
3127	}
3128	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3129	{
3130	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3131	fInterceptMovDRx = true;
3132	}
3133
3134	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3135	if (fInterceptDB)
3136	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3137	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3138	{
3139	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3140	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3141	#endif
3142	}
3143
3144	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3145	if (fInterceptMovDRx)
3146	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3147	else
3148	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3149
3150	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3151	AssertRCReturn(rc, rc);
3152	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3153	AssertRCReturn(rc, rc);
3154
3155	/* The guest's view of its DR7 is unblemished. */
3156	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3157	AssertRCReturn(rc, rc);
3158
3159	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3160	return rc;
3161	}
3162
3163
3164	#ifdef VBOX_STRICT
3165	/**
3166	* Strict function to validate segment registers.
3167	*
3168	* @remarks Requires CR0.
3169	*/
3170	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3171	{
3172	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3173	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3174	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3175
3176	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3177	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3178	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3179	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3180	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3181	{
3182	/* Protected mode checks */
3183	/* CS */
3184	Assert(pCtx->cs.Attr.n.u1Present);
3185	Assert(!(pCtx->cs.Attr.u & 0xf00));
3186	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3187	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3188	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3189	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3190	\|\| (pCtx->cs.Attr.n.u1Granularity));
3191	/* CS cannot be loaded with NULL in protected mode. */
3192	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3193	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3194	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3195	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3196	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3197	else
3198	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3199	/* SS */
3200	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3201	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3202	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3203	if ( !(pCtx->cr0 & X86_CR0_PE)
3204	\|\| pCtx->cs.Attr.n.u4Type == 3)
3205	{
3206	Assert(!pCtx->ss.Attr.n.u2Dpl);
3207	}
3208	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3209	{
3210	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3211	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3212	Assert(pCtx->ss.Attr.n.u1Present);
3213	Assert(!(pCtx->ss.Attr.u & 0xf00));
3214	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3215	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3216	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3217	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3218	\|\| (pCtx->ss.Attr.n.u1Granularity));
3219	}
3220	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3221	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3222	{
3223	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3224	Assert(pCtx->ds.Attr.n.u1Present);
3225	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3226	Assert(!(pCtx->ds.Attr.u & 0xf00));
3227	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3228	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3229	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3230	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3231	\|\| (pCtx->ds.Attr.n.u1Granularity));
3232	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3233	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3234	}
3235	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3236	{
3237	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3238	Assert(pCtx->es.Attr.n.u1Present);
3239	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3240	Assert(!(pCtx->es.Attr.u & 0xf00));
3241	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3242	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3243	\|\| !(pCtx->es.Attr.n.u1Granularity));
3244	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3245	\|\| (pCtx->es.Attr.n.u1Granularity));
3246	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3247	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3248	}
3249	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3250	{
3251	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3252	Assert(pCtx->fs.Attr.n.u1Present);
3253	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3254	Assert(!(pCtx->fs.Attr.u & 0xf00));
3255	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3256	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3257	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3258	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3259	\|\| (pCtx->fs.Attr.n.u1Granularity));
3260	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3261	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3262	}
3263	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3264	{
3265	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3266	Assert(pCtx->gs.Attr.n.u1Present);
3267	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3268	Assert(!(pCtx->gs.Attr.u & 0xf00));
3269	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3270	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3271	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3272	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3273	\|\| (pCtx->gs.Attr.n.u1Granularity));
3274	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3275	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3276	}
3277	/* 64-bit capable CPUs. */
3278	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3279	Assert(!(pCtx->cs.u64Base >> 32));
3280	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3281	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3282	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3283	# endif
3284	}
3285	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3286	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3287	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3288	{
3289	/* Real and v86 mode checks. */
3290	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3291	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3292	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3293	{
3294	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3295	}
3296	else
3297	{
3298	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3299	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3300	}
3301
3302	/* CS */
3303	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3304	Assert(pCtx->cs.u32Limit == 0xffff);
3305	Assert(u32CSAttr == 0xf3);
3306	/* SS */
3307	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3308	Assert(pCtx->ss.u32Limit == 0xffff);
3309	Assert(u32SSAttr == 0xf3);
3310	/* DS */
3311	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3312	Assert(pCtx->ds.u32Limit == 0xffff);
3313	Assert(u32DSAttr == 0xf3);
3314	/* ES */
3315	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3316	Assert(pCtx->es.u32Limit == 0xffff);
3317	Assert(u32ESAttr == 0xf3);
3318	/* FS */
3319	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3320	Assert(pCtx->fs.u32Limit == 0xffff);
3321	Assert(u32FSAttr == 0xf3);
3322	/* GS */
3323	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3324	Assert(pCtx->gs.u32Limit == 0xffff);
3325	Assert(u32GSAttr == 0xf3);
3326	/* 64-bit capable CPUs. */
3327	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3328	Assert(!(pCtx->cs.u64Base >> 32));
3329	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3330	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3331	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3332	# endif
3333	}
3334	}
3335	#endif /* VBOX_STRICT */
3336
3337
3338	/**
3339	* Writes a guest segment register into the guest-state area in the VMCS.
3340	*
3341	* @returns VBox status code.
3342	* @param pVCpu Pointer to the VMCPU.
3343	* @param idxSel Index of the selector in the VMCS.
3344	* @param idxLimit Index of the segment limit in the VMCS.
3345	* @param idxBase Index of the segment base in the VMCS.
3346	* @param idxAccess Index of the access rights of the segment in the VMCS.
3347	* @param pSelReg Pointer to the segment selector.
3348	* @param pCtx Pointer to the guest-CPU context.
3349	*
3350	* @remarks No-long-jump zone!!!
3351	*/
3352	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3353	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3354	{
3355	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3356	AssertRCReturn(rc, rc);
3357	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3358	AssertRCReturn(rc, rc);
3359	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3360	AssertRCReturn(rc, rc);
3361
3362	uint32_t u32Access = pSelReg->Attr.u;
3363	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3364	{
3365	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3366	u32Access = 0xf3;
3367	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3368	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3369	}
3370	else
3371	{
3372	/*
3373	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3374	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3375	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3376	* loaded in protected-mode have their attribute as 0.
3377	*/
3378	if (!u32Access)
3379	u32Access = HMVMX_SEL_UNUSABLE;
3380	}
3381
3382	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3383	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3384	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3385
3386	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3387	AssertRCReturn(rc, rc);
3388	return rc;
3389	}
3390
3391
3392	/**
3393	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3394	* into the guest-state area in the VMCS.
3395	*
3396	* @returns VBox status code.
3397	* @param pVM Pointer to the VM.
3398	* @param pVCPU Pointer to the VMCPU.
3399	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3400	* out-of-sync. Make sure to update the required fields
3401	* before using them.
3402	*
3403	* @remarks Requires CR0 (strict builds validation).
3404	* @remarks No-long-jump zone!!!
3405	*/
3406	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3407	{
3408	int rc = VERR_INTERNAL_ERROR_5;
3409	PVM pVM = pVCpu->CTX_SUFF(pVM);
3410
3411	/*
3412	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3413	*/
3414	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3415	{
3416	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3417	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3418	{
3419	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3420	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3421	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3422	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3423	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3424	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3425	}
3426
3427	#ifdef VBOX_WITH_REM
3428	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3429	{
3430	Assert(pVM->hm.s.vmx.pRealModeTSS);
3431	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3432	if ( pVCpu->hm.s.vmx.fWasInRealMode
3433	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3434	{
3435	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3436	in real-mode (e.g. OpenBSD 4.0) */
3437	REMFlushTBs(pVM);
3438	Log(("Load: Switch to protected mode detected!\n"));
3439	pVCpu->hm.s.vmx.fWasInRealMode = false;
3440	}
3441	}
3442	#endif
3443	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3444	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3445	AssertRCReturn(rc, rc);
3446	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3447	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3448	AssertRCReturn(rc, rc);
3449	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3450	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3451	AssertRCReturn(rc, rc);
3452	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3453	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3454	AssertRCReturn(rc, rc);
3455	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3456	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3457	AssertRCReturn(rc, rc);
3458	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3459	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3460	AssertRCReturn(rc, rc);
3461
3462	#ifdef VBOX_STRICT
3463	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3464	#endif
3465	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3466	}
3467
3468	/*
3469	* Guest TR.
3470	*/
3471	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3472	{
3473	/*
3474	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3475	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3476	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3477	*/
3478	uint16_t u16Sel = 0;
3479	uint32_t u32Limit = 0;
3480	uint64_t u64Base = 0;
3481	uint32_t u32AccessRights = 0;
3482
3483	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3484	{
3485	u16Sel = pMixedCtx->tr.Sel;
3486	u32Limit = pMixedCtx->tr.u32Limit;
3487	u64Base = pMixedCtx->tr.u64Base;
3488	u32AccessRights = pMixedCtx->tr.Attr.u;
3489	}
3490	else
3491	{
3492	Assert(pVM->hm.s.vmx.pRealModeTSS);
3493	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3494
3495	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3496	RTGCPHYS GCPhys;
3497	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3498	AssertRCReturn(rc, rc);
3499
3500	X86DESCATTR DescAttr;
3501	DescAttr.u = 0;
3502	DescAttr.n.u1Present = 1;
3503	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3504
3505	u16Sel = 0;
3506	u32Limit = HM_VTX_TSS_SIZE;
3507	u64Base = GCPhys; /* in real-mode phys = virt. */
3508	u32AccessRights = DescAttr.u;
3509	}
3510
3511	/* Validate. */
3512	Assert(!(u16Sel & RT_BIT(2)));
3513	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3514	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3515	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3516	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3517	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3518	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3519	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3520	Assert( (u32Limit & 0xfff) == 0xfff
3521	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3522	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3523	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3524
3525	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3526	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3527	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3528	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3529
3530	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3531	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3532	}
3533
3534	/*
3535	* Guest GDTR.
3536	*/
3537	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3538	{
3539	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3540	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3541
3542	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3543	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3544	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3545	}
3546
3547	/*
3548	* Guest LDTR.
3549	*/
3550	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3551	{
3552	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3553	uint32_t u32Access = 0;
3554	if (!pMixedCtx->ldtr.Attr.u)
3555	u32Access = HMVMX_SEL_UNUSABLE;
3556	else
3557	u32Access = pMixedCtx->ldtr.Attr.u;
3558
3559	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3560	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3561	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3562	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3563
3564	/* Validate. */
3565	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3566	{
3567	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3568	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3569	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3570	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3571	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3572	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3573	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3574	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3575	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3576	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3577	}
3578
3579	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3580	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3581	}
3582
3583	/*
3584	* Guest IDTR.
3585	*/
3586	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3587	{
3588	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3589	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3590
3591	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3592	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3593	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3594	}
3595
3596	return VINF_SUCCESS;
3597	}
3598
3599
3600	/**
3601	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3602	* areas. These MSRs will automatically be loaded to the host CPU on every
3603	* successful VM entry and stored from the host CPU on every successful VM exit.
3604	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3605	*
3606	* @returns VBox status code.
3607	* @param pVCpu Pointer to the VMCPU.
3608	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3609	* out-of-sync. Make sure to update the required fields
3610	* before using them.
3611	*
3612	* @remarks No-long-jump zone!!!
3613	*/
3614	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3615	{
3616	AssertPtr(pVCpu);
3617	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3618
3619	/*
3620	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3621	*/
3622	int rc = VINF_SUCCESS;
3623	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3624	{
3625	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3628	uint32_t cGuestMsrs = 0;
3629
3630	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3631	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3632	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3633	if (fSupportsNX \|\| fSupportsLongMode)
3634	{
3635	/** @todo support save IA32_EFER, i.e.
3636	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3637	* guest EFER need not be part of the VM-entry MSR-load area. */
3638	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3639	pGuestMsr->u32Reserved = 0;
3640	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3641	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3642	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3643	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3644	pGuestMsr++; cGuestMsrs++;
3645	if (fSupportsLongMode)
3646	{
3647	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3648	pGuestMsr->u32Reserved = 0;
3649	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3650	pGuestMsr++; cGuestMsrs++;
3651	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3652	pGuestMsr->u32Reserved = 0;
3653	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3654	pGuestMsr++; cGuestMsrs++;
3655	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3656	pGuestMsr->u32Reserved = 0;
3657	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3658	pGuestMsr++; cGuestMsrs++;
3659	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3660	pGuestMsr->u32Reserved = 0;
3661	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3662	pGuestMsr++; cGuestMsrs++;
3663	}
3664	}
3665
3666	/*
3667	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3668	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3669	*/
3670	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3671	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3672	{
3673	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3674	pGuestMsr->u32Reserved = 0;
3675	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3676	AssertRCReturn(rc, rc);
3677	pGuestMsr++; cGuestMsrs++;
3678	}
3679
3680	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3681	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3682	{
3683	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3684	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3685	}
3686
3687	/* Update the VCPU's copy of the guest MSR count. */
3688	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3690	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3691	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3692
3693	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3694	}
3695
3696	/*
3697	* Guest Sysenter MSRs.
3698	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3699	* VM-exits on WRMSRs for these MSRs.
3700	*/
3701	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3702	{
3703	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3704	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3705	}
3706	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3707	{
3708	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3709	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3710	}
3711	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3712	{
3713	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3714	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3715	}
3716
3717	return rc;
3718	}
3719
3720
3721	/**
3722	* Loads the guest activity state into the guest-state area in the VMCS.
3723	*
3724	* @returns VBox status code.
3725	* @param pVCpu Pointer to the VMCPU.
3726	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3727	* out-of-sync. Make sure to update the required fields
3728	* before using them.
3729	*
3730	* @remarks No-long-jump zone!!!
3731	*/
3732	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3733	{
3734	/** @todo See if we can make use of other states, e.g.
3735	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3736	int rc = VINF_SUCCESS;
3737	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3738	{
3739	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3740	AssertRCReturn(rc, rc);
3741	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3742	}
3743	return rc;
3744	}
3745
3746
3747	/**
3748	* Sets up the appropriate function to run guest code.
3749	*
3750	* @returns VBox status code.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3753	* out-of-sync. Make sure to update the required fields
3754	* before using them.
3755	*
3756	* @remarks No-long-jump zone!!!
3757	*/
3758	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3759	{
3760	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3761	{
3762	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3763	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3764	#endif
3765	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3766	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3767	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3768	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3769	#else
3770	/* 64-bit host or hybrid host. */
3771	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3772	#endif
3773	}
3774	else
3775	{
3776	/* Guest is not in long mode, use the 32-bit handler. */
3777	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3778	}
3779	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3780	return VINF_SUCCESS;
3781	}
3782
3783
3784	/**
3785	* Wrapper for running the guest code in VT-x.
3786	*
3787	* @returns VBox strict status code.
3788	* @param pVM Pointer to the VM.
3789	* @param pVCpu Pointer to the VMCPU.
3790	* @param pCtx Pointer to the guest-CPU context.
3791	*
3792	* @remarks No-long-jump zone!!!
3793	*/
3794	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3795	{
3796	/*
3797	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3798	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3799	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3800	*/
3801	#ifdef VBOX_WITH_KERNEL_USING_XMM
3802	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3803	#else
3804	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3805	#endif
3806	}
3807
3808
3809	/**
3810	* Report world-switch error and dump some useful debug info.
3811	*
3812	* @param pVM Pointer to the VM.
3813	* @param pVCpu Pointer to the VMCPU.
3814	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3815	* @param pCtx Pointer to the guest-CPU context.
3816	* @param pVmxTransient Pointer to the VMX transient structure (only
3817	* exitReason updated).
3818	*/
3819	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3820	{
3821	Assert(pVM);
3822	Assert(pVCpu);
3823	Assert(pCtx);
3824	Assert(pVmxTransient);
3825	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3826
3827	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3828	switch (rcVMRun)
3829	{
3830	case VERR_VMX_INVALID_VMXON_PTR:
3831	AssertFailed();
3832	break;
3833	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3834	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3835	{
3836	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3837	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3838	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3839	AssertRC(rc);
3840
3841	#ifdef VBOX_STRICT
3842	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3843	pVmxTransient->uExitReason));
3844	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3845	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3846	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3847	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3848	else
3849	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3850
3851	/* VMX control bits. */
3852	uint32_t u32Val;
3853	uint64_t u64Val;
3854	HMVMXHCUINTREG uHCReg;
3855	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3856	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3857	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3858	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3859	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3860	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3861	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3862	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3863	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3864	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3865	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3866	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3867	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3868	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3869	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3870	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3871	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3872	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3873	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3874	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3875	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3876	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3877	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3878	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3879	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3880	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3881	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3882	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3883	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3884	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3885	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3886	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3887	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3888	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3889	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3890	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3891	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3892	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3893	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3894	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3895	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3896	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3897
3898	/* Guest bits. */
3899	RTGCUINTREG uGCReg;
3900	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3901	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3902	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3903	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3904	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3905	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3906	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3907	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3908
3909	/* Host bits. */
3910	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3911	Log(("Host CR0 %#RHr\n", uHCReg));
3912	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3913	Log(("Host CR3 %#RHr\n", uHCReg));
3914	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3915	Log(("Host CR4 %#RHr\n", uHCReg));
3916
3917	RTGDTR HostGdtr;
3918	PCX86DESCHC pDesc;
3919	ASMGetGDTR(&HostGdtr);
3920	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3921	Log(("Host CS %#08x\n", u32Val));
3922	if (u32Val < HostGdtr.cbGdt)
3923	{
3924	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3925	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3926	}
3927
3928	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3929	Log(("Host DS %#08x\n", u32Val));
3930	if (u32Val < HostGdtr.cbGdt)
3931	{
3932	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3933	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3934	}
3935
3936	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3937	Log(("Host ES %#08x\n", u32Val));
3938	if (u32Val < HostGdtr.cbGdt)
3939	{
3940	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3941	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3942	}
3943
3944	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3945	Log(("Host FS %#08x\n", u32Val));
3946	if (u32Val < HostGdtr.cbGdt)
3947	{
3948	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3949	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3950	}
3951
3952	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3953	Log(("Host GS %#08x\n", u32Val));
3954	if (u32Val < HostGdtr.cbGdt)
3955	{
3956	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3957	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3958	}
3959
3960	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3961	Log(("Host SS %#08x\n", u32Val));
3962	if (u32Val < HostGdtr.cbGdt)
3963	{
3964	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3965	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3966	}
3967
3968	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3969	Log(("Host TR %#08x\n", u32Val));
3970	if (u32Val < HostGdtr.cbGdt)
3971	{
3972	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3973	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3974	}
3975
3976	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3977	Log(("Host TR Base %#RHv\n", uHCReg));
3978	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3979	Log(("Host GDTR Base %#RHv\n", uHCReg));
3980	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3981	Log(("Host IDTR Base %#RHv\n", uHCReg));
3982	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3983	Log(("Host SYSENTER CS %#08x\n", u32Val));
3984	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3985	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3986	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3987	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3988	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3989	Log(("Host RSP %#RHv\n", uHCReg));
3990	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3991	Log(("Host RIP %#RHv\n", uHCReg));
3992	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3993	if (HMVMX_IS_64BIT_HOST_MODE())
3994	{
3995	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3996	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3997	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3998	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3999	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4000	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4001	}
4002	# endif
4003	#endif /* VBOX_STRICT */
4004	break;
4005	}
4006
4007	default:
4008	/* Impossible */
4009	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4010	break;
4011	}
4012	NOREF(pVM);
4013	}
4014
4015
4016	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4017	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4018	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4019	#endif
4020	#ifdef VBOX_STRICT
4021	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4022	{
4023	switch (idxField)
4024	{
4025	case VMX_VMCS_GUEST_RIP:
4026	case VMX_VMCS_GUEST_RSP:
4027	case VMX_VMCS_GUEST_DR7:
4028	case VMX_VMCS_GUEST_SYSENTER_EIP:
4029	case VMX_VMCS_GUEST_SYSENTER_ESP:
4030	case VMX_VMCS_GUEST_GDTR_BASE:
4031	case VMX_VMCS_GUEST_IDTR_BASE:
4032	case VMX_VMCS_GUEST_CS_BASE:
4033	case VMX_VMCS_GUEST_DS_BASE:
4034	case VMX_VMCS_GUEST_ES_BASE:
4035	case VMX_VMCS_GUEST_FS_BASE:
4036	case VMX_VMCS_GUEST_GS_BASE:
4037	case VMX_VMCS_GUEST_SS_BASE:
4038	case VMX_VMCS_GUEST_LDTR_BASE:
4039	case VMX_VMCS_GUEST_TR_BASE:
4040	case VMX_VMCS_GUEST_CR3:
4041	return true;
4042	}
4043	return false;
4044	}
4045
4046	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4047	{
4048	switch (idxField)
4049	{
4050	/* Read-only fields. */
4051	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4052	return true;
4053	}
4054	/* Remaining readable fields should also be writable. */
4055	return hmR0VmxIsValidWriteField(idxField);
4056	}
4057	#endif /* VBOX_STRICT */
4058
4059	/**
4060	* Executes the specified handler in 64-bit mode.
4061	*
4062	* @returns VBox status code.
4063	* @param pVM Pointer to the VM.
4064	* @param pVCpu Pointer to the VMCPU.
4065	* @param pCtx Pointer to the guest CPU context.
4066	* @param enmOp The operation to perform.
4067	* @param cbParam Number of parameters.
4068	* @param paParam Array of 32-bit parameters.
4069	*/
4070	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4071	uint32_t *paParam)
4072	{
4073	int rc, rc2;
4074	PHMGLOBLCPUINFO pCpu;
4075	RTHCPHYS HCPhysCpuPage;
4076	RTCCUINTREG uOldEFlags;
4077
4078	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4079	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4080	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4081	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4082
4083	#ifdef VBOX_STRICT
4084	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4085	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4086
4087	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4088	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4089	#endif
4090
4091	/* Disable interrupts. */
4092	uOldEFlags = ASMIntDisableFlags();
4093
4094	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4095	RTCPUID idHostCpu = RTMpCpuId();
4096	CPUMR0SetLApic(pVM, idHostCpu);
4097	#endif
4098
4099	pCpu = HMR0GetCurrentCpu();
4100	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4101
4102	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4103	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4104
4105	/* Leave VMX Root Mode. */
4106	VMXDisable();
4107
4108	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4109
4110	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4111	CPUMSetHyperEIP(pVCpu, enmOp);
4112	for (int i = (int)cbParam - 1; i >= 0; i--)
4113	CPUMPushHyper(pVCpu, paParam[i]);
4114
4115	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4116
4117	/* Call the switcher. */
4118	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4119	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4120
4121	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4122	/* Make sure the VMX instructions don't cause #UD faults. */
4123	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4124
4125	/* Re-enter VMX Root Mode */
4126	rc2 = VMXEnable(HCPhysCpuPage);
4127	if (RT_FAILURE(rc2))
4128	{
4129	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4130	ASMSetFlags(uOldEFlags);
4131	return rc2;
4132	}
4133
4134	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4135	AssertRC(rc2);
4136	Assert(!(ASMGetFlags() & X86_EFL_IF));
4137	ASMSetFlags(uOldEFlags);
4138	return rc;
4139	}
4140
4141
4142	/**
4143	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4144	* supporting 64-bit guests.
4145	*
4146	* @returns VBox status code.
4147	* @param fResume Whether to VMLAUNCH or VMRESUME.
4148	* @param pCtx Pointer to the guest-CPU context.
4149	* @param pCache Pointer to the VMCS cache.
4150	* @param pVM Pointer to the VM.
4151	* @param pVCpu Pointer to the VMCPU.
4152	*/
4153	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4154	{
4155	uint32_t aParam[6];
4156	PHMGLOBLCPUINFO pCpu = NULL;
4157	RTHCPHYS HCPhysCpuPage = 0;
4158	int rc = VERR_INTERNAL_ERROR_5;
4159
4160	pCpu = HMR0GetCurrentCpu();
4161	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4162
4163	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4164	pCache->uPos = 1;
4165	pCache->interPD = PGMGetInterPaeCR3(pVM);
4166	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4167	#endif
4168
4169	#ifdef VBOX_STRICT
4170	pCache->TestIn.HCPhysCpuPage = 0;
4171	pCache->TestIn.HCPhysVmcs = 0;
4172	pCache->TestIn.pCache = 0;
4173	pCache->TestOut.HCPhysVmcs = 0;
4174	pCache->TestOut.pCache = 0;
4175	pCache->TestOut.pCtx = 0;
4176	pCache->TestOut.eflags = 0;
4177	#endif
4178
4179	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4180	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4181	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4182	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4183	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4184	aParam[5] = 0;
4185
4186	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4187	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4188	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4189	#endif
4190	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4191
4192	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4193	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4194	Assert(pCtx->dr[4] == 10);
4195	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4196	#endif
4197
4198	#ifdef VBOX_STRICT
4199	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4200	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4201	pVCpu->hm.s.vmx.HCPhysVmcs));
4202	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4203	pCache->TestOut.HCPhysVmcs));
4204	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4205	pCache->TestOut.pCache));
4206	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4207	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4208	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4209	pCache->TestOut.pCtx));
4210	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4211	#endif
4212	return rc;
4213	}
4214
4215
4216	/**
4217	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4218	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4219	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4220	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4221	*
4222	* @returns VBox status code.
4223	* @param pVM Pointer to the VM.
4224	* @param pVCpu Pointer to the VMCPU.
4225	*/
4226	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4227	{
4228	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4229	{ \
4230	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4231	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4232	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4233	++cReadFields; \
4234	}
4235
4236	AssertPtr(pVM);
4237	AssertPtr(pVCpu);
4238	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4239	uint32_t cReadFields = 0;
4240
4241	/* Guest-natural selector base fields */
4242	#if 0
4243	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4245	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4246	#endif
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4252	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4257	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4260	#if 0
4261	/* Unused natural width guest-state fields. */
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4264	#endif
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4267
4268	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4269	#if 0
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4279	#endif
4280
4281	/* Natural width guest-state fields. */
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4283	#if 0
4284	/* Currently unused field. */
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4286	#endif
4287
4288	if (pVM->hm.s.fNestedPaging)
4289	{
4290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4291	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4292	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4293	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4294	}
4295	else
4296	{
4297	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4298	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4299	}
4300
4301	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4302	return VINF_SUCCESS;
4303	}
4304
4305
4306	/**
4307	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4308	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4309	* darwin, running 64-bit guests).
4310	*
4311	* @returns VBox status code.
4312	* @param pVCpu Pointer to the VMCPU.
4313	* @param idxField The VMCS field encoding.
4314	* @param u64Val 16, 32 or 64 bits value.
4315	*/
4316	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4317	{
4318	int rc;
4319	switch (idxField)
4320	{
4321	/*
4322	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4323	*/
4324	/* 64-bit Control fields. */
4325	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4326	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4327	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4328	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4329	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4330	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4331	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4332	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4333	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4334	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4335	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4336	case VMX_VMCS64_CTRL_EPTP_FULL:
4337	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4338	/* 64-bit Guest-state fields. */
4339	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4340	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4341	case VMX_VMCS64_GUEST_PAT_FULL:
4342	case VMX_VMCS64_GUEST_EFER_FULL:
4343	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4344	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4345	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4346	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4347	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4348	/* 64-bit Host-state fields. */
4349	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4350	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4351	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4352	{
4353	rc = VMXWriteVmcs32(idxField, u64Val);
4354	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4355	break;
4356	}
4357
4358	/*
4359	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4360	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4361	*/
4362	/* Natural-width Guest-state fields. */
4363	case VMX_VMCS_GUEST_CR3:
4364	case VMX_VMCS_GUEST_ES_BASE:
4365	case VMX_VMCS_GUEST_CS_BASE:
4366	case VMX_VMCS_GUEST_SS_BASE:
4367	case VMX_VMCS_GUEST_DS_BASE:
4368	case VMX_VMCS_GUEST_FS_BASE:
4369	case VMX_VMCS_GUEST_GS_BASE:
4370	case VMX_VMCS_GUEST_LDTR_BASE:
4371	case VMX_VMCS_GUEST_TR_BASE:
4372	case VMX_VMCS_GUEST_GDTR_BASE:
4373	case VMX_VMCS_GUEST_IDTR_BASE:
4374	case VMX_VMCS_GUEST_DR7:
4375	case VMX_VMCS_GUEST_RSP:
4376	case VMX_VMCS_GUEST_RIP:
4377	case VMX_VMCS_GUEST_SYSENTER_ESP:
4378	case VMX_VMCS_GUEST_SYSENTER_EIP:
4379	{
4380	if (!(u64Val >> 32))
4381	{
4382	/* If this field is 64-bit, VT-x will zero out the top bits. */
4383	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4384	}
4385	else
4386	{
4387	/* Assert that only the 32->64 switcher case should ever come here. */
4388	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4389	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4390	}
4391	break;
4392	}
4393
4394	default:
4395	{
4396	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4397	rc = VERR_INVALID_PARAMETER;
4398	break;
4399	}
4400	}
4401	AssertRCReturn(rc, rc);
4402	return rc;
4403	}
4404
4405
4406	/**
4407	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4408	* hosts (except darwin) for 64-bit guests.
4409	*
4410	* @param pVCpu Pointer to the VMCPU.
4411	* @param idxField The VMCS field encoding.
4412	* @param u64Val 16, 32 or 64 bits value.
4413	*/
4414	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4415	{
4416	AssertPtr(pVCpu);
4417	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4418
4419	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4420	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4421
4422	/* Make sure there are no duplicates. */
4423	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4424	{
4425	if (pCache->Write.aField[i] == idxField)
4426	{
4427	pCache->Write.aFieldVal[i] = u64Val;
4428	return VINF_SUCCESS;
4429	}
4430	}
4431
4432	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4433	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4434	pCache->Write.cValidEntries++;
4435	return VINF_SUCCESS;
4436	}
4437
4438	/* Enable later when the assembly code uses these as callbacks. */
4439	#if 0
4440	/*
4441	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4442	*
4443	* @param pVCpu Pointer to the VMCPU.
4444	* @param pCache Pointer to the VMCS cache.
4445	*
4446	* @remarks No-long-jump zone!!!
4447	*/
4448	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4449	{
4450	AssertPtr(pCache);
4451	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4452	{
4453	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4454	AssertRC(rc);
4455	}
4456	pCache->Write.cValidEntries = 0;
4457	}
4458
4459
4460	/**
4461	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4462	*
4463	* @param pVCpu Pointer to the VMCPU.
4464	* @param pCache Pointer to the VMCS cache.
4465	*
4466	* @remarks No-long-jump zone!!!
4467	*/
4468	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4469	{
4470	AssertPtr(pCache);
4471	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4472	{
4473	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4474	AssertRC(rc);
4475	}
4476	}
4477	#endif
4478	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4479
4480
4481	/**
4482	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4483	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4484	* timer.
4485	*
4486	* @returns VBox status code.
4487	* @param pVCpu Pointer to the VMCPU.
4488	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4489	* out-of-sync. Make sure to update the required fields
4490	* before using them.
4491	* @remarks No-long-jump zone!!!
4492	*/
4493	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4494	{
4495	int rc = VERR_INTERNAL_ERROR_5;
4496	bool fOffsettedTsc = false;
4497	PVM pVM = pVCpu->CTX_SUFF(pVM);
4498	if (pVM->hm.s.vmx.fUsePreemptTimer)
4499	{
4500	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4501
4502	/* Make sure the returned values have sane upper and lower boundaries. */
4503	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4504	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4505	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4506	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4507
4508	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4509	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4510	}
4511	else
4512	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4513
4514	if (fOffsettedTsc)
4515	{
4516	uint64_t u64CurTSC = ASMReadTSC();
4517	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4518	{
4519	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4520	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4521
4522	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4523	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4524	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4525	}
4526	else
4527	{
4528	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4529	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4530	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4531	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4532	}
4533	}
4534	else
4535	{
4536	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4537	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4538	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4539	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4540	}
4541	}
4542
4543
4544	/**
4545	* Determines if an exception is a contributory exception. Contributory
4546	* exceptions are ones which can cause double-faults. Page-fault is
4547	* intentionally not included here as it's a conditional contributory exception.
4548	*
4549	* @returns true if the exception is contributory, false otherwise.
4550	* @param uVector The exception vector.
4551	*/
4552	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4553	{
4554	switch (uVector)
4555	{
4556	case X86_XCPT_GP:
4557	case X86_XCPT_SS:
4558	case X86_XCPT_NP:
4559	case X86_XCPT_TS:
4560	case X86_XCPT_DE:
4561	return true;
4562	default:
4563	break;
4564	}
4565	return false;
4566	}
4567
4568
4569	/**
4570	* Sets an event as a pending event to be injected into the guest.
4571	*
4572	* @param pVCpu Pointer to the VMCPU.
4573	* @param u32IntrInfo The VM-entry interruption-information field.
4574	* @param cbInstr The VM-entry instruction length in bytes (for software
4575	* interrupts, exceptions and privileged software
4576	* exceptions).
4577	* @param u32ErrCode The VM-entry exception error code.
4578	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4579	* page-fault.
4580	*/
4581	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4582	RTGCUINTPTR GCPtrFaultAddress)
4583	{
4584	Assert(!pVCpu->hm.s.Event.fPending);
4585	pVCpu->hm.s.Event.fPending = true;
4586	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4587	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4588	pVCpu->hm.s.Event.cbInstr = cbInstr;
4589	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4590	}
4591
4592
4593	/**
4594	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4595	*
4596	* @param pVCpu Pointer to the VMCPU.
4597	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4598	* out-of-sync. Make sure to update the required fields
4599	* before using them.
4600	*/
4601	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4602	{
4603	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4604	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4605	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4606	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4607	}
4608
4609
4610	/**
4611	* Handle a condition that occurred while delivering an event through the guest
4612	* IDT.
4613	*
4614	* @returns VBox status code (informational error codes included).
4615	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4616	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4617	* continue execution of the guest which will delivery the #DF.
4618	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4619	*
4620	* @param pVCpu Pointer to the VMCPU.
4621	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4622	* out-of-sync. Make sure to update the required fields
4623	* before using them.
4624	* @param pVmxTransient Pointer to the VMX transient structure.
4625	*
4626	* @remarks No-long-jump zone!!!
4627	*/
4628	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4629	{
4630	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4631	AssertRC(rc);
4632	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4633	{
4634	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4635	AssertRCReturn(rc, rc);
4636
4637	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4638	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4639	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4640
4641	typedef enum
4642	{
4643	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4644	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4645	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4646	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4647	} VMXREFLECTXCPT;
4648
4649	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4650	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4651	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4652	{
4653	enmReflect = VMXREFLECTXCPT_XCPT;
4654	#ifdef VBOX_STRICT
4655	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4656	&& uExitVector == X86_XCPT_PF)
4657	{
4658	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4659	}
4660	#endif
4661	if ( uExitVector == X86_XCPT_PF
4662	&& uIdtVector == X86_XCPT_PF)
4663	{
4664	pVmxTransient->fVectoringPF = true;
4665	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4666	}
4667	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4668	&& hmR0VmxIsContributoryXcpt(uExitVector)
4669	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4670	\|\| uIdtVector == X86_XCPT_PF))
4671	{
4672	enmReflect = VMXREFLECTXCPT_DF;
4673	}
4674	else if (uIdtVector == X86_XCPT_DF)
4675	enmReflect = VMXREFLECTXCPT_TF;
4676	}
4677	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4678	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4679	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4680	{
4681	/*
4682	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4683	* (whatever they are) as they reoccur when restarting the instruction.
4684	*/
4685	enmReflect = VMXREFLECTXCPT_XCPT;
4686	}
4687
4688	switch (enmReflect)
4689	{
4690	case VMXREFLECTXCPT_XCPT:
4691	{
4692	uint32_t u32ErrCode = 0;
4693	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4694	{
4695	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4696	AssertRCReturn(rc, rc);
4697	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4698	}
4699
4700	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4701	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4702	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4703	rc = VINF_SUCCESS;
4704	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4705	pVCpu->hm.s.Event.u32ErrCode));
4706	break;
4707	}
4708
4709	case VMXREFLECTXCPT_DF:
4710	{
4711	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4712	rc = VINF_VMX_DOUBLE_FAULT;
4713	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4714	uIdtVector, uExitVector));
4715	break;
4716	}
4717
4718	case VMXREFLECTXCPT_TF:
4719	{
4720	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4721	rc = VINF_EM_RESET;
4722	break;
4723	}
4724
4725	default:
4726	Assert(rc == VINF_SUCCESS);
4727	break;
4728	}
4729	}
4730	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4731	return rc;
4732	}
4733
4734
4735	/**
4736	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4737	*
4738	* @returns VBox status code.
4739	* @param pVCpu Pointer to the VMCPU.
4740	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4741	* out-of-sync. Make sure to update the required fields
4742	* before using them.
4743	*
4744	* @remarks No-long-jump zone!!!
4745	*/
4746	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4747	{
4748	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4749	{
4750	uint32_t uVal = 0;
4751	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4752	AssertRCReturn(rc, rc);
4753	uint32_t uShadow = 0;
4754	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4755	AssertRCReturn(rc, rc);
4756
4757	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4758	CPUMSetGuestCR0(pVCpu, uVal);
4759	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4760	}
4761	return VINF_SUCCESS;
4762	}
4763
4764
4765	/**
4766	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4767	*
4768	* @returns VBox status code.
4769	* @param pVCpu Pointer to the VMCPU.
4770	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4771	* out-of-sync. Make sure to update the required fields
4772	* before using them.
4773	*
4774	* @remarks No-long-jump zone!!!
4775	*/
4776	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4777	{
4778	int rc = VINF_SUCCESS;
4779	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4780	{
4781	uint32_t uVal = 0;
4782	uint32_t uShadow = 0;
4783	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4784	AssertRCReturn(rc, rc);
4785	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4786	AssertRCReturn(rc, rc);
4787
4788	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4789	CPUMSetGuestCR4(pVCpu, uVal);
4790	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4791	}
4792	return rc;
4793	}
4794
4795
4796	/**
4797	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4798	*
4799	* @returns VBox status code.
4800	* @param pVCpu Pointer to the VMCPU.
4801	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4802	* out-of-sync. Make sure to update the required fields
4803	* before using them.
4804	*
4805	* @remarks No-long-jump zone!!!
4806	*/
4807	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4808	{
4809	int rc = VINF_SUCCESS;
4810	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4811	{
4812	RTGCUINTREG uVal = 0;
4813	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4814	AssertRCReturn(rc, rc);
4815
4816	pMixedCtx->rip = uVal;
4817	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4818	}
4819	return rc;
4820	}
4821
4822
4823	/**
4824	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4825	*
4826	* @returns VBox status code.
4827	* @param pVCpu Pointer to the VMCPU.
4828	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4829	* out-of-sync. Make sure to update the required fields
4830	* before using them.
4831	*
4832	* @remarks No-long-jump zone!!!
4833	*/
4834	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4835	{
4836	int rc = VINF_SUCCESS;
4837	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4838	{
4839	RTGCUINTREG uVal = 0;
4840	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4841	AssertRCReturn(rc, rc);
4842
4843	pMixedCtx->rsp = uVal;
4844	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4845	}
4846	return rc;
4847	}
4848
4849
4850	/**
4851	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4852	*
4853	* @returns VBox status code.
4854	* @param pVCpu Pointer to the VMCPU.
4855	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4856	* out-of-sync. Make sure to update the required fields
4857	* before using them.
4858	*
4859	* @remarks No-long-jump zone!!!
4860	*/
4861	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4862	{
4863	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4864	{
4865	uint32_t uVal = 0;
4866	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4867	AssertRCReturn(rc, rc);
4868
4869	pMixedCtx->eflags.u32 = uVal;
4870	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4871	{
4872	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4873	Log(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4874
4875	pMixedCtx->eflags.Bits.u1VM = 0;
4876	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4877	}
4878
4879	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4880	}
4881	return VINF_SUCCESS;
4882	}
4883
4884
4885	/**
4886	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4887	* guest-CPU context.
4888	*/
4889	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4890	{
4891	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4892	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4893	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4894	return rc;
4895	}
4896
4897
4898	/**
4899	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4900	* from the guest-state area in the VMCS.
4901	*
4902	* @param pVCpu Pointer to the VMCPU.
4903	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4904	* out-of-sync. Make sure to update the required fields
4905	* before using them.
4906	*
4907	* @remarks No-long-jump zone!!!
4908	*/
4909	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4910	{
4911	uint32_t uIntrState = 0;
4912	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4913	AssertRC(rc);
4914
4915	if (!uIntrState)
4916	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4917	else
4918	{
4919	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4920	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4921	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4922	AssertRC(rc);
4923	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4924	AssertRC(rc);
4925
4926	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4927	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4928	}
4929	}
4930
4931
4932	/**
4933	* Saves the guest's activity state.
4934	*
4935	* @returns VBox status code.
4936	* @param pVCpu Pointer to the VMCPU.
4937	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4938	* out-of-sync. Make sure to update the required fields
4939	* before using them.
4940	*
4941	* @remarks No-long-jump zone!!!
4942	*/
4943	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4944	{
4945	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4946	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4947	return VINF_SUCCESS;
4948	}
4949
4950
4951	/**
4952	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4953	* the current VMCS into the guest-CPU context.
4954	*
4955	* @returns VBox status code.
4956	* @param pVCpu Pointer to the VMCPU.
4957	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4958	* out-of-sync. Make sure to update the required fields
4959	* before using them.
4960	*
4961	* @remarks No-long-jump zone!!!
4962	*/
4963	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4964	{
4965	int rc = VINF_SUCCESS;
4966	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4967	{
4968	uint32_t u32Val = 0;
4969	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4970	pMixedCtx->SysEnter.cs = u32Val;
4971	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4972	}
4973
4974	RTGCUINTREG uGCVal = 0;
4975	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4976	{
4977	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4978	pMixedCtx->SysEnter.eip = uGCVal;
4979	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4980	}
4981	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4982	{
4983	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4984	pMixedCtx->SysEnter.esp = uGCVal;
4985	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4986	}
4987	return rc;
4988	}
4989
4990
4991	/**
4992	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4993	* context.
4994	*
4995	* @returns VBox status code.
4996	* @param pVCpu Pointer to the VMCPU.
4997	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4998	* out-of-sync. Make sure to update the required fields
4999	* before using them.
5000	*
5001	* @remarks No-long-jump zone!!!
5002	*/
5003	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5004	{
5005	int rc = VINF_SUCCESS;
5006	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5007	{
5008	RTGCUINTREG uVal = 0;
5009	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5010	pMixedCtx->fs.u64Base = uVal;
5011	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5012	}
5013	return rc;
5014	}
5015
5016
5017	/**
5018	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5019	* context.
5020	*
5021	* @returns VBox status code.
5022	* @param pVCpu Pointer to the VMCPU.
5023	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5024	* out-of-sync. Make sure to update the required fields
5025	* before using them.
5026	*
5027	* @remarks No-long-jump zone!!!
5028	*/
5029	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5030	{
5031	int rc = VINF_SUCCESS;
5032	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5033	{
5034	RTGCUINTREG uVal = 0;
5035	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5036	pMixedCtx->gs.u64Base = uVal;
5037	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5038	}
5039	return rc;
5040	}
5041
5042
5043	/**
5044	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5045	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5046	* and TSC_AUX.
5047	*
5048	* @returns VBox status code.
5049	* @param pVCpu Pointer to the VMCPU.
5050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5051	* out-of-sync. Make sure to update the required fields
5052	* before using them.
5053	*
5054	* @remarks No-long-jump zone!!!
5055	*/
5056	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5057	{
5058	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5059	return VINF_SUCCESS;
5060
5061	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5062	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5063	{
5064	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5065	pMsr += i;
5066	switch (pMsr->u32IndexMSR)
5067	{
5068	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5069	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5070	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5071	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5072	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5073	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5074	default:
5075	{
5076	AssertFailed();
5077	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5078	}
5079	}
5080	}
5081	#endif
5082
5083	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5084	return VINF_SUCCESS;
5085	}
5086
5087
5088	/**
5089	* Saves the guest control registers from the current VMCS into the guest-CPU
5090	* context.
5091	*
5092	* @returns VBox status code.
5093	* @param pVCpu Pointer to the VMCPU.
5094	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5095	* out-of-sync. Make sure to update the required fields
5096	* before using them.
5097	*
5098	* @remarks No-long-jump zone!!!
5099	*/
5100	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5101	{
5102	/* Guest CR0. Guest FPU. */
5103	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5104	AssertRCReturn(rc, rc);
5105
5106	/* Guest CR4. */
5107	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5108	AssertRCReturn(rc, rc);
5109
5110	/* Guest CR2 - updated always during the world-switch or in #PF. */
5111	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5112	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5113	{
5114	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5115	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5116
5117	PVM pVM = pVCpu->CTX_SUFF(pVM);
5118	if ( pVM->hm.s.fNestedPaging
5119	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5120	{
5121	RTGCUINTREG uVal = 0;
5122	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5123	if (pMixedCtx->cr3 != uVal)
5124	{
5125	CPUMSetGuestCR3(pVCpu, uVal);
5126	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5127	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5128	}
5129
5130	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5131	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5132	{
5133	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5134	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5135	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5136	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5137	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5138	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5139	}
5140	}
5141
5142	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5143	}
5144	return rc;
5145	}
5146
5147
5148	/**
5149	* Reads a guest segment register from the current VMCS into the guest-CPU
5150	* context.
5151	*
5152	* @returns VBox status code.
5153	* @param pVCpu Pointer to the VMCPU.
5154	* @param idxSel Index of the selector in the VMCS.
5155	* @param idxLimit Index of the segment limit in the VMCS.
5156	* @param idxBase Index of the segment base in the VMCS.
5157	* @param idxAccess Index of the access rights of the segment in the VMCS.
5158	* @param pSelReg Pointer to the segment selector.
5159	*
5160	* @remarks No-long-jump zone!!!
5161	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5162	* as that takes care of whether to read from the VMCS cache or not.
5163	*/
5164	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5165	PCPUMSELREG pSelReg)
5166	{
5167	uint32_t u32Val = 0;
5168	int rc = VMXReadVmcs32(idxSel, &u32Val);
5169	AssertRCReturn(rc, rc);
5170	pSelReg->Sel = (uint16_t)u32Val;
5171	pSelReg->ValidSel = (uint16_t)u32Val;
5172	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5173
5174	rc = VMXReadVmcs32(idxLimit, &u32Val);
5175	AssertRCReturn(rc, rc);
5176	pSelReg->u32Limit = u32Val;
5177
5178	RTGCUINTREG uGCVal = 0;
5179	rc = VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5180	AssertRCReturn(rc, rc);
5181	pSelReg->u64Base = uGCVal;
5182
5183	rc = VMXReadVmcs32(idxAccess, &u32Val);
5184	AssertRCReturn(rc, rc);
5185	pSelReg->Attr.u = u32Val;
5186
5187	/*
5188	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5189	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5190	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5191	*
5192	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5193	*/
5194	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5195	{
5196	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5197	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5198	}
5199	return VINF_SUCCESS;
5200	}
5201
5202	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5203	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5204	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5205	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5206	#else
5207	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5208	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5209	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5210	#endif
5211
5212	/**
5213	* Saves the guest segment registers from the current VMCS into the guest-CPU
5214	* context.
5215	*
5216	* @returns VBox status code.
5217	* @param pVCpu Pointer to the VMCPU.
5218	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5219	* out-of-sync. Make sure to update the required fields
5220	* before using them.
5221	*
5222	* @remarks No-long-jump zone!!!
5223	*/
5224	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5225	{
5226	/* Guest segment registers. */
5227	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5228	{
5229	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5230	AssertRCReturn(rc, rc);
5231	rc = VMXLOCAL_READ_SEG(CS, cs);
5232	AssertRCReturn(rc, rc);
5233	rc = VMXLOCAL_READ_SEG(SS, ss);
5234	AssertRCReturn(rc, rc);
5235	rc = VMXLOCAL_READ_SEG(DS, ds);
5236	AssertRCReturn(rc, rc);
5237	rc = VMXLOCAL_READ_SEG(ES, es);
5238	AssertRCReturn(rc, rc);
5239	rc = VMXLOCAL_READ_SEG(FS, fs);
5240	AssertRCReturn(rc, rc);
5241	rc = VMXLOCAL_READ_SEG(GS, gs);
5242	AssertRCReturn(rc, rc);
5243
5244	/* Restore segment attributes for real-on-v86 mode hack. */
5245	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5246	{
5247	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5248	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5249	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5250	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5251	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5252	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5253	}
5254	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5255	}
5256
5257	return VINF_SUCCESS;
5258	}
5259
5260
5261	/**
5262	* Saves the guest descriptor table registers and task register from the current
5263	* VMCS into the guest-CPU context.
5264	*
5265	* @returns VBox status code.
5266	* @param pVCpu Pointer to the VMCPU.
5267	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5268	* out-of-sync. Make sure to update the required fields
5269	* before using them.
5270	*
5271	* @remarks No-long-jump zone!!!
5272	*/
5273	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5274	{
5275	int rc = VINF_SUCCESS;
5276
5277	/* Guest LDTR. */
5278	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5279	{
5280	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5281	AssertRCReturn(rc, rc);
5282	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5283	}
5284
5285	/* Guest GDTR. */
5286	RTGCUINTREG uGCVal = 0;
5287	uint32_t u32Val = 0;
5288	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5289	{
5290	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal); AssertRCReturn(rc, rc);
5291	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5292	pMixedCtx->gdtr.pGdt = uGCVal;
5293	pMixedCtx->gdtr.cbGdt = u32Val;
5294	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5295	}
5296
5297	/* Guest IDTR. */
5298	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5299	{
5300	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal); AssertRCReturn(rc, rc);
5301	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5302	pMixedCtx->idtr.pIdt = uGCVal;
5303	pMixedCtx->idtr.cbIdt = u32Val;
5304	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5305	}
5306
5307	/* Guest TR. */
5308	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5309	{
5310	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5311	AssertRCReturn(rc, rc);
5312
5313	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5314	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5315	{
5316	rc = VMXLOCAL_READ_SEG(TR, tr);
5317	AssertRCReturn(rc, rc);
5318	}
5319	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5320	}
5321	return rc;
5322	}
5323
5324	#undef VMXLOCAL_READ_SEG
5325
5326
5327	/**
5328	* Saves the guest debug registers from the current VMCS into the guest-CPU
5329	* context.
5330	*
5331	* @returns VBox status code.
5332	* @param pVCpu Pointer to the VMCPU.
5333	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5334	* out-of-sync. Make sure to update the required fields
5335	* before using them.
5336	*
5337	* @remarks No-long-jump zone!!!
5338	*/
5339	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5340	{
5341	int rc = VINF_SUCCESS;
5342	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5343	{
5344	RTGCUINTREG uVal;
5345	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5346	pMixedCtx->dr[7] = uVal;
5347
5348	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5349	}
5350	return rc;
5351	}
5352
5353
5354	/**
5355	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5356	*
5357	* @returns VBox status code.
5358	* @param pVCpu Pointer to the VMCPU.
5359	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5360	* out-of-sync. Make sure to update the required fields
5361	* before using them.
5362	*
5363	* @remarks No-long-jump zone!!!
5364	*/
5365	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5366	{
5367	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5368	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5369	return VINF_SUCCESS;
5370	}
5371
5372
5373	/**
5374	* Saves the entire guest state from the currently active VMCS into the
5375	* guest-CPU context. This essentially VMREADs all guest-data.
5376	*
5377	* @returns VBox status code.
5378	* @param pVCpu Pointer to the VMCPU.
5379	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5380	* out-of-sync. Make sure to update the required fields
5381	* before using them.
5382	*/
5383	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5384	{
5385	Assert(pVCpu);
5386	Assert(pMixedCtx);
5387
5388	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5389	return VINF_SUCCESS;
5390
5391	VMMRZCallRing3Disable(pVCpu);
5392	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5393	LogFunc(("\n"));
5394
5395	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5396	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5397
5398	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5399	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5400
5401	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5402	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5403
5404	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5405	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5406
5407	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5408	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5409
5410	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5411	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5412
5413	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5414	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5415
5416	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5417	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5418
5419	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5420	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5421
5422	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5423	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5424
5425	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5427
5428	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5429	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5430
5431	VMMRZCallRing3Enable(pVCpu);
5432	return rc;
5433	}
5434
5435
5436	/**
5437	* Check per-VM and per-VCPU force flag actions that require us to go back to
5438	* ring-3 for one reason or another.
5439	*
5440	* @returns VBox status code (information status code included).
5441	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5442	* ring-3.
5443	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5444	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5445	* interrupts)
5446	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5447	* all EMTs to be in ring-3.
5448	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5449	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5450	* to the EM loop.
5451	*
5452	* @param pVM Pointer to the VM.
5453	* @param pVCpu Pointer to the VMCPU.
5454	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5455	* out-of-sync. Make sure to update the required fields
5456	* before using them.
5457	*/
5458	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5459	{
5460	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5461
5462	int rc = VERR_INTERNAL_ERROR_5;
5463	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5464	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5465	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5466	{
5467	/* We need the control registers now, make sure the guest-CPU context is updated. */
5468	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5469	AssertRCReturn(rc, rc);
5470
5471	/* Pending HM CR3 sync. */
5472	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5473	{
5474	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5475	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5476	}
5477	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5478	{
5479	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5480	AssertRC(rc);
5481	}
5482
5483	/* Pending PGM C3 sync. */
5484	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5485	{
5486	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5487	if (rc != VINF_SUCCESS)
5488	{
5489	AssertRC(rc);
5490	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5491	return rc;
5492	}
5493	}
5494
5495	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5496	/* -XXX- what was that about single stepping? */
5497	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5498	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5499	{
5500	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5501	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5502	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5503	return rc;
5504	}
5505
5506	/* Pending VM request packets, such as hardware interrupts. */
5507	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5508	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5509	{
5510	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5511	return VINF_EM_PENDING_REQUEST;
5512	}
5513
5514	/* Pending PGM pool flushes. */
5515	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5516	{
5517	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5518	return VINF_PGM_POOL_FLUSH_PENDING;
5519	}
5520
5521	/* Pending DMA requests. */
5522	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5523	{
5524	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5525	return VINF_EM_RAW_TO_R3;
5526	}
5527	}
5528
5529	/* Paranoia. */
5530	Assert(rc != VERR_EM_INTERPRETER);
5531	return VINF_SUCCESS;
5532	}
5533
5534
5535	/**
5536	* Converts any TRPM trap into a pending VMX event. This is typically used when
5537	* entering from ring-3 (not longjmp returns).
5538	*
5539	* @param pVCpu Pointer to the VMCPU.
5540	*/
5541	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5542	{
5543	Assert(TRPMHasTrap(pVCpu));
5544	Assert(!pVCpu->hm.s.Event.fPending);
5545
5546	uint8_t uVector;
5547	TRPMEVENT enmTrpmEvent;
5548	RTGCUINT uErrCode;
5549	RTGCUINTPTR GCPtrFaultAddress;
5550	uint8_t cbInstr;
5551
5552	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5553	AssertRC(rc);
5554
5555	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5556	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5557	if (enmTrpmEvent == TRPM_TRAP)
5558	{
5559	switch (uVector)
5560	{
5561	case X86_XCPT_BP:
5562	case X86_XCPT_OF:
5563	{
5564	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5565	break;
5566	}
5567
5568	case X86_XCPT_PF:
5569	case X86_XCPT_DF:
5570	case X86_XCPT_TS:
5571	case X86_XCPT_NP:
5572	case X86_XCPT_SS:
5573	case X86_XCPT_GP:
5574	case X86_XCPT_AC:
5575	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5576	/* no break! */
5577	default:
5578	{
5579	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5580	break;
5581	}
5582	}
5583	}
5584	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5585	{
5586	if (uVector != X86_XCPT_NMI)
5587	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5588	else
5589	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5590	}
5591	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5592	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5593	else
5594	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5595
5596	rc = TRPMResetTrap(pVCpu);
5597	AssertRC(rc);
5598	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5599	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5600	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5601	}
5602
5603
5604	/**
5605	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5606	* VT-x to execute any instruction.
5607	*
5608	* @param pvCpu Pointer to the VMCPU.
5609	*/
5610	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5611	{
5612	Assert(pVCpu->hm.s.Event.fPending);
5613
5614	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5615	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5616	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5617	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5618
5619	/* If a trap was already pending, we did something wrong! */
5620	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5621
5622	TRPMEVENT enmTrapType;
5623	switch (uVectorType)
5624	{
5625	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5626	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5627	enmTrapType = TRPM_HARDWARE_INT;
5628	break;
5629	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5630	enmTrapType = TRPM_SOFTWARE_INT;
5631	break;
5632	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5633	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5634	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5635	enmTrapType = TRPM_TRAP;
5636	break;
5637	default:
5638	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5639	enmTrapType = TRPM_32BIT_HACK;
5640	break;
5641	}
5642
5643	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5644	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5645	AssertRC(rc);
5646
5647	if (fErrorCodeValid)
5648	TRPMSetErrorCode(pVCpu, uErrorCode);
5649	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5650	&& uVector == X86_XCPT_PF)
5651	{
5652	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5653	}
5654	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5655	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5656	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5657	{
5658	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5659	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5660	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5661	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5662	}
5663	pVCpu->hm.s.Event.fPending = false;
5664	}
5665
5666
5667	/**
5668	* Does the necessary state syncing before doing a longjmp to ring-3.
5669	*
5670	* @param pVM Pointer to the VM.
5671	* @param pVCpu Pointer to the VMCPU.
5672	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5673	* out-of-sync. Make sure to update the required fields
5674	* before using them.
5675	* @param rcExit The reason for exiting to ring-3. Can be
5676	* VINF_VMM_UNKNOWN_RING3_CALL.
5677	*
5678	* @remarks No-long-jmp zone!!!
5679	*/
5680	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5681	{
5682	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5683	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5684
5685	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5686	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5687	AssertRC(rc);
5688
5689	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5690	if (CPUMIsGuestFPUStateActive(pVCpu))
5691	{
5692	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5693	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5694	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5695	}
5696
5697	/* Restore debug registers if necessary and resync on next R0 reentry. */
5698	if (CPUMIsGuestDebugStateActive(pVCpu))
5699	{
5700	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5701	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5702	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5703	}
5704	else if (CPUMIsHyperDebugStateActive(pVCpu))
5705	{
5706	CPUMR0LoadHostDebugState(pVM, pVCpu);
5707	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5708	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5709	}
5710
5711	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5712	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5713	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5714	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5715	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5716	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5717	}
5718
5719
5720	/**
5721	* An action requires us to go back to ring-3. This function does the necessary
5722	* steps before we can safely return to ring-3. This is not the same as longjmps
5723	* to ring-3, this is voluntary.
5724	*
5725	* @param pVM Pointer to the VM.
5726	* @param pVCpu Pointer to the VMCPU.
5727	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5728	* out-of-sync. Make sure to update the required fields
5729	* before using them.
5730	* @param rcExit The reason for exiting to ring-3. Can be
5731	* VINF_VMM_UNKNOWN_RING3_CALL.
5732	*/
5733	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5734	{
5735	Assert(pVM);
5736	Assert(pVCpu);
5737	Assert(pMixedCtx);
5738	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5739
5740	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5741	{
5742	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5743	return;
5744	}
5745	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5746	{
5747	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5748	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5749	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5750	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5751	return;
5752	}
5753
5754	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5755	VMMRZCallRing3Disable(pVCpu);
5756	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5757
5758	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5759	if (pVCpu->hm.s.Event.fPending)
5760	{
5761	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5762	Assert(!pVCpu->hm.s.Event.fPending);
5763	}
5764
5765	/* Sync. the guest state. */
5766	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5767	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5768
5769	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5770	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5771	\| CPUM_CHANGED_LDTR
5772	\| CPUM_CHANGED_GDTR
5773	\| CPUM_CHANGED_IDTR
5774	\| CPUM_CHANGED_TR
5775	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5776
5777	/* On our way back from ring-3 the following needs to be done. */
5778	/** @todo This can change with preemption hooks. */
5779	if (rcExit == VINF_EM_RAW_INTERRUPT)
5780	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5781	else
5782	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5783
5784	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5785	VMMRZCallRing3Enable(pVCpu);
5786	}
5787
5788
5789	/**
5790	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5791	* longjump to ring-3 and possibly get preempted.
5792	*
5793	* @param pVCpu Pointer to the VMCPU.
5794	* @param enmOperation The operation causing the ring-3 longjump.
5795	* @param pvUser The user argument (pointer to the possibly
5796	* out-of-date guest-CPU context).
5797	*
5798	* @remarks Must never be called with @a enmOperation ==
5799	* VMMCALLRING3_VM_R0_ASSERTION.
5800	*/
5801	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5802	{
5803	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5804	Assert(pVCpu);
5805	Assert(pvUser);
5806	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5807	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5808
5809	VMMRZCallRing3Disable(pVCpu);
5810	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5811	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5812	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5813	VMMRZCallRing3Enable(pVCpu);
5814	}
5815
5816
5817	/**
5818	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5819	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5820	*
5821	* @returns VBox status code.
5822	* @param pVCpu Pointer to the VMCPU.
5823	*/
5824	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5825	{
5826	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5827	{
5828	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5829	{
5830	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5831	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5832	AssertRC(rc);
5833	}
5834	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5835	}
5836
5837
5838	/**
5839	* Injects any pending events into the guest if the guest is in a state to
5840	* receive them.
5841	*
5842	* @returns VBox status code (informational status codes included).
5843	* @param pVCpu Pointer to the VMCPU.
5844	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5845	* out-of-sync. Make sure to update the required fields
5846	* before using them.
5847	*/
5848	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5849	{
5850	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5851	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5852	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5853	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5854
5855	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5856	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5857	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5858	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5859	Assert(!TRPMHasTrap(pVCpu));
5860
5861	int rc = VINF_SUCCESS;
5862	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5863	{
5864	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5865	bool fInject = true;
5866	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5867	{
5868	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5869	AssertRCReturn(rc, rc);
5870	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5871	if ( fBlockInt
5872	\|\| fBlockSti
5873	\|\| fBlockMovSS)
5874	{
5875	fInject = false;
5876	}
5877	}
5878	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5879	&& ( fBlockMovSS
5880	\|\| fBlockSti))
5881	{
5882	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5883	fInject = false;
5884	}
5885
5886	if (fInject)
5887	{
5888	Log(("Injecting pending event\n"));
5889	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5890	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5891	AssertRCReturn(rc, rc);
5892	pVCpu->hm.s.Event.fPending = false;
5893	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5894	}
5895	else
5896	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5897	} /** @todo SMI. SMIs take priority over NMIs. */
5898	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5899	{
5900	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5901	if ( !fBlockMovSS
5902	&& !fBlockSti)
5903	{
5904	Log(("Injecting NMI\n"));
5905	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5906	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5907	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5908	0 /* GCPtrFaultAddress */, &uIntrState);
5909	AssertRCReturn(rc, rc);
5910	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5911	}
5912	else
5913	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5914	}
5915	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5916	{
5917	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5918	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5919	AssertRCReturn(rc, rc);
5920	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5921	if ( !fBlockInt
5922	&& !fBlockSti
5923	&& !fBlockMovSS)
5924	{
5925	uint8_t u8Interrupt;
5926	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5927	if (RT_SUCCESS(rc))
5928	{
5929	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5930	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5931	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5932	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5933	0 /* GCPtrFaultAddress */, &uIntrState);
5934	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5935	}
5936	else
5937	{
5938	/** @todo Does this actually happen? If not turn it into an assertion. */
5939	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5940	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5941	rc = VINF_SUCCESS;
5942	}
5943	}
5944	else
5945	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5946	}
5947
5948	/*
5949	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5950	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5951	*/
5952	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5953	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5954	int rc2 = VINF_SUCCESS;
5955	if ( fBlockSti
5956	\|\| fBlockMovSS)
5957	{
5958	if (!DBGFIsStepping(pVCpu))
5959	{
5960	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5961	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5962	{
5963	/*
5964	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5965	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5966	*/
5967	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5968	AssertRCReturn(rc, rc);
5969	}
5970	}
5971	else
5972	{
5973	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5974	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5975	uIntrState = 0;
5976	}
5977	}
5978
5979	/*
5980	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5981	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5982	*/
5983	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5984	AssertRC(rc2);
5985
5986	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5987	return rc;
5988	}
5989
5990
5991	/**
5992	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5993	*
5994	* @param pVCpu Pointer to the VMCPU.
5995	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5996	* out-of-sync. Make sure to update the required fields
5997	* before using them.
5998	*/
5999	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6000	{
6001	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6002	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6003	}
6004
6005
6006	/**
6007	* Injects a double-fault (#DF) exception into the VM.
6008	*
6009	* @returns VBox status code (informational status code included).
6010	* @param pVCpu Pointer to the VMCPU.
6011	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6012	* out-of-sync. Make sure to update the required fields
6013	* before using them.
6014	*/
6015	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6016	{
6017	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6018	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6019	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6020	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6021	puIntrState);
6022	}
6023
6024
6025	/**
6026	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6027	*
6028	* @param pVCpu Pointer to the VMCPU.
6029	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6030	* out-of-sync. Make sure to update the required fields
6031	* before using them.
6032	*/
6033	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6034	{
6035	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6036	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6037	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6038	}
6039
6040
6041	/**
6042	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6043	*
6044	* @param pVCpu Pointer to the VMCPU.
6045	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6046	* out-of-sync. Make sure to update the required fields
6047	* before using them.
6048	* @param cbInstr The value of RIP that is to be pushed on the guest
6049	* stack.
6050	*/
6051	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6052	{
6053	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6054	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6055	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6056	}
6057
6058
6059	/**
6060	* Injects a general-protection (#GP) fault into the VM.
6061	*
6062	* @returns VBox status code (informational status code included).
6063	* @param pVCpu Pointer to the VMCPU.
6064	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6065	* out-of-sync. Make sure to update the required fields
6066	* before using them.
6067	* @param u32ErrorCode The error code associated with the #GP.
6068	*/
6069	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6070	uint32_t *puIntrState)
6071	{
6072	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6073	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6074	if (fErrorCodeValid)
6075	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6076	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6077	puIntrState);
6078	}
6079
6080
6081	/**
6082	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6083	*
6084	* @param pVCpu Pointer to the VMCPU.
6085	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6086	* out-of-sync. Make sure to update the required fields
6087	* before using them.
6088	* @param uVector The software interrupt vector number.
6089	* @param cbInstr The value of RIP that is to be pushed on the guest
6090	* stack.
6091	*/
6092	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6093	{
6094	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6095	if ( uVector == X86_XCPT_BP
6096	\|\| uVector == X86_XCPT_OF)
6097	{
6098	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6099	}
6100	else
6101	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6102	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6103	}
6104
6105
6106	/**
6107	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6108	* stack.
6109	*
6110	* @returns VBox status code (information status code included).
6111	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6112	* @param pVM Pointer to the VM.
6113	* @param pMixedCtx Pointer to the guest-CPU context.
6114	* @param uValue The value to push to the guest stack.
6115	*/
6116	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6117	{
6118	/*
6119	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6120	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6121	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6122	*/
6123	if (pMixedCtx->sp == 1)
6124	return VINF_EM_RESET;
6125	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6126	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6127	AssertRCReturn(rc, rc);
6128	return rc;
6129	}
6130
6131
6132	/**
6133	* Injects an event into the guest upon VM-entry by updating the relevant fields
6134	* in the VM-entry area in the VMCS.
6135	*
6136	* @returns VBox status code (informational error codes included).
6137	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6138	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6139	*
6140	* @param pVCpu Pointer to the VMCPU.
6141	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6142	* be out-of-sync. Make sure to update the required
6143	* fields before using them.
6144	* @param u64IntrInfo The VM-entry interruption-information field.
6145	* @param cbInstr The VM-entry instruction length in bytes (for
6146	* software interrupts, exceptions and privileged
6147	* software exceptions).
6148	* @param u32ErrCode The VM-entry exception error code.
6149	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6150	* @param puIntrState Pointer to the current guest interruptibility-state.
6151	* This interruptibility-state will be updated if
6152	* necessary. This cannot not be NULL.
6153	*
6154	* @remarks No-long-jump zone!!!
6155	* @remarks Requires CR0!
6156	*/
6157	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6158	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6159	{
6160	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6161	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6162	Assert(puIntrState);
6163	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6164
6165	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6166	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6167
6168	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6169	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6170	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6171
6172	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6173
6174	/* We require CR0 to check if the guest is in real-mode. */
6175	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6176	AssertRCReturn(rc, rc);
6177
6178	/*
6179	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6180	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6181	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6182	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6183	*/
6184	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6185	{
6186	PVM pVM = pVCpu->CTX_SUFF(pVM);
6187	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6188	{
6189	Assert(PDMVmmDevHeapIsEnabled(pVM));
6190	Assert(pVM->hm.s.vmx.pRealModeTSS);
6191
6192	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6193	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6194	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6195	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6196	AssertRCReturn(rc, rc);
6197	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6198
6199	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6200	const size_t cbIdtEntry = 4;
6201	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6202	{
6203	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6204	if (uVector == X86_XCPT_DF)
6205	return VINF_EM_RESET;
6206	else if (uVector == X86_XCPT_GP)
6207	{
6208	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6209	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6210	}
6211
6212	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6213	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6214	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6215	}
6216
6217	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6218	uint16_t uGuestIp = pMixedCtx->ip;
6219	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6220	{
6221	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6222	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6223	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6224	}
6225	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6226	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6227
6228	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6229	uint16_t offIdtEntry = 0;
6230	RTSEL selIdtEntry = 0;
6231	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6232	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6233	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6234	AssertRCReturn(rc, rc);
6235
6236	/* Construct the stack frame for the interrupt/exception handler. */
6237	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6238	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6239	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6240	AssertRCReturn(rc, rc);
6241
6242	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6243	if (rc == VINF_SUCCESS)
6244	{
6245	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6246	pMixedCtx->rip = offIdtEntry;
6247	pMixedCtx->cs.Sel = selIdtEntry;
6248	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6249	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6250	&& uVector == X86_XCPT_PF)
6251	{
6252	pMixedCtx->cr2 = GCPtrFaultAddress;
6253	}
6254	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6255	\| HM_CHANGED_GUEST_RIP
6256	\| HM_CHANGED_GUEST_RFLAGS
6257	\| HM_CHANGED_GUEST_RSP;
6258
6259	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6260	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6261	{
6262	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6263	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6264	Log(("Clearing inhibition due to STI.\n"));
6265	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6266	}
6267	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6268	}
6269	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6270	return rc;
6271	}
6272	else
6273	{
6274	/*
6275	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6276	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6277	*/
6278	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6279	}
6280	}
6281
6282	/* Validate. */
6283	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6284	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6285	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6286
6287	/* Inject. */
6288	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6289	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6290	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6291	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6292
6293	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6294	&& uVector == X86_XCPT_PF)
6295	{
6296	pMixedCtx->cr2 = GCPtrFaultAddress;
6297	}
6298	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6299
6300	AssertRCReturn(rc, rc);
6301	return rc;
6302	}
6303
6304
6305	/**
6306	* Enters the VT-x session.
6307	*
6308	* @returns VBox status code.
6309	* @param pVM Pointer to the VM.
6310	* @param pVCpu Pointer to the VMCPU.
6311	* @param pCpu Pointer to the CPU info struct.
6312	*/
6313	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6314	{
6315	AssertPtr(pVM);
6316	AssertPtr(pVCpu);
6317	Assert(pVM->hm.s.vmx.fSupported);
6318	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6319	NOREF(pCpu);
6320
6321	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6322
6323	/* Make sure we're in VMX root mode. */
6324	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6325	if (!(u32HostCR4 & X86_CR4_VMXE))
6326	{
6327	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6328	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6329	}
6330
6331	/* Load the active VMCS as the current one. */
6332	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6333	if (RT_FAILURE(rc))
6334	return rc;
6335
6336	/** @todo this will change with preemption hooks where can can VMRESUME as long
6337	* as we're no preempted. */
6338	pVCpu->hm.s.fResumeVM = false;
6339	return VINF_SUCCESS;
6340	}
6341
6342
6343	/**
6344	* Leaves the VT-x session.
6345	*
6346	* @returns VBox status code.
6347	* @param pVM Pointer to the VM.
6348	* @param pVCpu Pointer to the VMCPU.
6349	* @param pCtx Pointer to the guest-CPU context.
6350	*/
6351	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6352	{
6353	AssertPtr(pVCpu);
6354	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6355	NOREF(pVM);
6356	NOREF(pCtx);
6357
6358	/** @todo this will change with preemption hooks where we only VMCLEAR when
6359	* we are actually going to be preempted, not all the time like we
6360	* currently do. */
6361	/*
6362	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6363	* and mark the VMCS launch-state as "clear".
6364	*/
6365	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6366	return rc;
6367	}
6368
6369
6370	/**
6371	* Saves the host state in the VMCS host-state.
6372	* Sets up the VM-exit MSR-load area.
6373	*
6374	* The CPU state will be loaded from these fields on every successful VM-exit.
6375	*
6376	* @returns VBox status code.
6377	* @param pVM Pointer to the VM.
6378	* @param pVCpu Pointer to the VMCPU.
6379	*
6380	* @remarks No-long-jump zone!!!
6381	*/
6382	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6383	{
6384	AssertPtr(pVM);
6385	AssertPtr(pVCpu);
6386	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6387
6388	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6389
6390	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6391	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6392	return VINF_SUCCESS;
6393
6394	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6395	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6396
6397	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6398	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6399
6400	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6401	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6402
6403	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6404	return rc;
6405	}
6406
6407
6408	/**
6409	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6410	* loaded from these fields on every successful VM-entry.
6411	*
6412	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6413	* Sets up the VM-entry controls.
6414	* Sets up the appropriate VMX non-root function to execute guest code based on
6415	* the guest CPU mode.
6416	*
6417	* @returns VBox status code.
6418	* @param pVM Pointer to the VM.
6419	* @param pVCpu Pointer to the VMCPU.
6420	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6421	* out-of-sync. Make sure to update the required fields
6422	* before using them.
6423	*
6424	* @remarks No-long-jump zone!!!
6425	*/
6426	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6427	{
6428	AssertPtr(pVM);
6429	AssertPtr(pVCpu);
6430	AssertPtr(pMixedCtx);
6431	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6432
6433	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6434
6435	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6436
6437	/* Determine real-on-v86 mode. */
6438	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6439	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6440	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6441	{
6442	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6443	}
6444
6445	/*
6446	* Load the guest-state into the VMCS.
6447	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6448	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6449	*/
6450	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6451	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6452
6453	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6455
6456	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6458
6459	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6461
6462	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6463	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6465
6466	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6467	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6468
6469	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6470	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6471
6472	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6473	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6474
6475	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6476	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6477
6478	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6479	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6480
6481	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6482	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6483	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6484
6485	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6486	return rc;
6487	}
6488
6489
6490	/**
6491	* Does the preparations before executing guest code in VT-x.
6492	*
6493	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6494	* recompiler. We must be cautious what we do here regarding committing
6495	* guest-state information into the the VMCS assuming we assuredly execute the
6496	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6497	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6498	* that the recompiler can (and should) use them when it resumes guest
6499	* execution. Otherwise such operations must be done when we can no longer
6500	* exit to ring-3.
6501	*
6502	* @returns VBox status code (informational status codes included).
6503	* @retval VINF_SUCCESS if we can proceed with running the guest.
6504	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6505	* into the guest.
6506	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6507	*
6508	* @param pVM Pointer to the VM.
6509	* @param pVCpu Pointer to the VMCPU.
6510	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6511	* out-of-sync. Make sure to update the required fields
6512	* before using them.
6513	* @param pVmxTransient Pointer to the VMX transient structure.
6514	*
6515	* @remarks Called with preemption disabled.
6516	*/
6517	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6518	{
6519	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6520
6521	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6522	PGMRZDynMapFlushAutoSet(pVCpu);
6523	#endif
6524
6525	/* Check force flag actions that might require us to go back to ring-3. */
6526	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6527	if (rc != VINF_SUCCESS)
6528	return rc;
6529
6530	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6531	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6532	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6533	{
6534	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6535	RTGCPHYS GCPhysApicBase;
6536	GCPhysApicBase = pMixedCtx->msrApicBase;
6537	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6538
6539	/* Unalias any existing mapping. */
6540	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6541	AssertRCReturn(rc, rc);
6542
6543	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6544	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6545	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6546	AssertRCReturn(rc, rc);
6547
6548	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6549	}
6550
6551	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6552	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6553	pVmxTransient->uEFlags = ASMIntDisableFlags();
6554	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6555	{
6556	ASMSetFlags(pVmxTransient->uEFlags);
6557	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6558	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6559	return VINF_EM_RAW_INTERRUPT;
6560	}
6561	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6562	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6563	#endif
6564
6565	/*
6566	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6567	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6568	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6569	*/
6570	/** @todo Rework event evaluation and injection to be complete separate. */
6571	if (TRPMHasTrap(pVCpu))
6572	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6573
6574	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6575	AssertRCReturn(rc, rc);
6576	return rc;
6577	}
6578
6579
6580	/**
6581	* Prepares to run guest code in VT-x and we've committed to doing so. This
6582	* means there is no backing out to ring-3 or anywhere else at this
6583	* point.
6584	*
6585	* @param pVM Pointer to the VM.
6586	* @param pVCpu Pointer to the VMCPU.
6587	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6588	* out-of-sync. Make sure to update the required fields
6589	* before using them.
6590	* @param pVmxTransient Pointer to the VMX transient structure.
6591	*
6592	* @remarks Called with preemption disabled.
6593	* @remarks No-long-jump zone!!!
6594	*/
6595	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6596	{
6597	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6598	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6599
6600	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6601	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6602	pVmxTransient->uEFlags = ASMIntDisableFlags();
6603	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6604	#endif
6605
6606	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6607	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6608	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6609	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6610	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6611	#endif
6612	int rc = VINF_SUCCESS;
6613	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6614	{
6615	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6616	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6617	}
6618	else if (pVCpu->hm.s.fContextUseFlags)
6619	{
6620	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6621	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6622	}
6623	AssertRC(rc);
6624	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6625
6626	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6627	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6628	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6629
6630	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6631	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6632	{
6633	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6634	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6635	}
6636
6637	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6638	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6639	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6640
6641	/*
6642	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6643	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6644	*/
6645	if (pVM->hm.s.fTPRPatchingActive)
6646	{
6647	Assert(!CPUMIsGuestInLongMode(pVCpu));
6648
6649	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6650	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6651	AssertRC(rc);
6652
6653	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6654	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6655	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6656	see hmR0VmxLoadGuestApicState(). */
6657	}
6658
6659	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6660	/*
6661	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6662	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6663	*/
6664	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6665	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6666	{
6667	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6668	uint64_t u64HostTscAux = 0;
6669	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6670	AssertRC(rc2);
6671	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6672	}
6673	#endif
6674
6675	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6676	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6677	to start executing. */
6678	}
6679
6680
6681	/**
6682	* Performs some essential restoration of state after running guest code in
6683	* VT-x.
6684	*
6685	* @param pVM Pointer to the VM.
6686	* @param pVCpu Pointer to the VMCPU.
6687	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6688	* out-of-sync. Make sure to update the required fields
6689	* before using them.
6690	* @param pVmxTransient Pointer to the VMX transient structure.
6691	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6692	*
6693	* @remarks Called with interrupts disabled.
6694	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6695	* unconditionally when it is safe to do so.
6696	*/
6697	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6698	{
6699	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6700	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6701
6702	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6703	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6704	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6705	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6706	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6707
6708	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6709	{
6710	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6711	/* Restore host's TSC_AUX. */
6712	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6713	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6714	#endif
6715	/** @todo Find a way to fix hardcoding a guestimate. */
6716	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6717	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6718	}
6719
6720	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6721	Assert(!(ASMGetFlags() & X86_EFL_IF));
6722	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6723
6724	/* Restore the effects of TPR patching if any. */
6725	if (pVM->hm.s.fTPRPatchingActive)
6726	{
6727	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6728	AssertRC(rc);
6729	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6730	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6731	}
6732
6733	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6734	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6735
6736	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6737	uint32_t uExitReason;
6738	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6739	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6740	AssertRC(rc);
6741	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6742	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6743
6744	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6745	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6746
6747	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6748	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6749	{
6750	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6751	return;
6752	}
6753
6754	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6755	{
6756	/* Update the guest interruptibility-state from the VMCS. */
6757	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6758	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6759	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6760	AssertRC(rc);
6761	#endif
6762	/*
6763	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6764	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6765	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6766	*/
6767	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6768	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6769	{
6770	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6771	AssertRC(rc);
6772	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6773	}
6774	}
6775	}
6776
6777
6778	/**
6779	* Runs the guest code using VT-x.
6780	*
6781	* @returns VBox status code.
6782	* @param pVM Pointer to the VM.
6783	* @param pVCpu Pointer to the VMCPU.
6784	* @param pCtx Pointer to the guest-CPU context.
6785	*
6786	* @remarks Called with preemption disabled.
6787	*/
6788	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6789	{
6790	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6791	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6792
6793	VMXTRANSIENT VmxTransient;
6794	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6795	int rc = VERR_INTERNAL_ERROR_5;
6796	uint32_t cLoops = 0;
6797
6798	for (;; cLoops++)
6799	{
6800	Assert(!HMR0SuspendPending());
6801	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6802	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6803	(unsigned)RTMpCpuId(), cLoops));
6804
6805	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6806	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6807	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6808	if (rc != VINF_SUCCESS)
6809	break;
6810
6811	/*
6812	* No longjmps to ring-3 from this point on!!!
6813	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6814	* This also disables flushing of the R0-logger instance (if any).
6815	*/
6816	VMMRZCallRing3Disable(pVCpu);
6817	VMMRZCallRing3RemoveNotification(pVCpu);
6818	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6819
6820	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6821	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6822
6823	/*
6824	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6825	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6826	*/
6827	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6828	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6829	{
6830	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6831	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6832	return rc;
6833	}
6834
6835	/* Handle the VM-exit. */
6836	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6837	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6838	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6839	HMVMX_START_EXIT_DISPATCH_PROF();
6840	#ifdef HMVMX_USE_FUNCTION_TABLE
6841	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6842	#else
6843	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6844	#endif
6845	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6846	if (rc != VINF_SUCCESS)
6847	break;
6848	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6849	{
6850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6851	rc = VINF_EM_RAW_INTERRUPT;
6852	break;
6853	}
6854	}
6855
6856	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6857	if (rc == VERR_EM_INTERPRETER)
6858	rc = VINF_EM_RAW_EMULATE_INSTR;
6859	else if (rc == VINF_EM_RESET)
6860	rc = VINF_EM_TRIPLE_FAULT;
6861	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6862	return rc;
6863	}
6864
6865
6866	#ifndef HMVMX_USE_FUNCTION_TABLE
6867	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6868	{
6869	int rc;
6870	switch (rcReason)
6871	{
6872	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6873	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6874	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6875	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6876	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6877	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6878	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6879	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6880	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6881	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6882	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6883	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6884	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6885	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6886	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6887	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6888	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6889	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6890	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6891	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6892	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6893	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6894	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6895	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6896	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6897	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6898	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6899	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6900	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6901	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6902	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6903	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6904	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6905
6906	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6907	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6908	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6909	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6910	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6911	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6912	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6913	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6914	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6915
6916	case VMX_EXIT_VMCALL:
6917	case VMX_EXIT_VMCLEAR:
6918	case VMX_EXIT_VMLAUNCH:
6919	case VMX_EXIT_VMPTRLD:
6920	case VMX_EXIT_VMPTRST:
6921	case VMX_EXIT_VMREAD:
6922	case VMX_EXIT_VMRESUME:
6923	case VMX_EXIT_VMWRITE:
6924	case VMX_EXIT_VMXOFF:
6925	case VMX_EXIT_VMXON:
6926	case VMX_EXIT_INVEPT:
6927	case VMX_EXIT_INVVPID:
6928	case VMX_EXIT_VMFUNC:
6929	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6930	break;
6931	default:
6932	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6933	break;
6934	}
6935	return rc;
6936	}
6937	#endif
6938
6939	#ifdef DEBUG
6940	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6941	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6942	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6943	# define VMX_ASSERT_PREEMPT_CPUID() \
6944	do \
6945	{ \
6946	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6947	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6948	} while (0)
6949
6950	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6951	do { \
6952	AssertPtr(pVCpu); \
6953	AssertPtr(pMixedCtx); \
6954	AssertPtr(pVmxTransient); \
6955	Assert(pVmxTransient->fVMEntryFailed == false); \
6956	Assert(ASMIntAreEnabled()); \
6957	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6958	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6959	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6960	(unsigned)pVCpu->idCpu)); \
6961	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6962	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6963	VMX_ASSERT_PREEMPT_CPUID(); \
6964	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6965	} while (0)
6966	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6967	do { \
6968	LogFunc(("\n")); \
6969	} while(0)
6970	#else /* Release builds */
6971	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6972	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6973	#endif
6974
6975
6976	/**
6977	* Advances the guest RIP after reading it from the VMCS.
6978	*
6979	* @returns VBox status code.
6980	* @param pVCpu Pointer to the VMCPU.
6981	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6982	* out-of-sync. Make sure to update the required fields
6983	* before using them.
6984	* @param pVmxTransient Pointer to the VMX transient structure.
6985	*
6986	* @remarks No-long-jump zone!!!
6987	*/
6988	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6989	{
6990	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6991	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6992	AssertRCReturn(rc, rc);
6993
6994	pMixedCtx->rip += pVmxTransient->cbInstr;
6995	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6996	return rc;
6997	}
6998
6999
7000	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7001	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7002	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7003	/**
7004	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7005	*/
7006	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7007	{
7008	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7010	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7011	Assert(ASMIntAreEnabled());
7012	return VINF_SUCCESS;
7013	#else
7014	return VINF_EM_RAW_INTERRUPT;
7015	#endif
7016	}
7017
7018
7019	/**
7020	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7021	*/
7022	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7023	{
7024	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7025	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7026
7027	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7028	AssertRCReturn(rc, rc);
7029
7030	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7031	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
7032	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7033
7034	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7035	{
7036	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7037	return VINF_EM_RAW_INTERRUPT;
7038	}
7039
7040	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7041	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7042	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7043	{
7044	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7045	return VINF_SUCCESS;
7046	}
7047	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7048	{
7049	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7050	return rc;
7051	}
7052
7053	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7054	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7055	switch (uIntrType)
7056	{
7057	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7058	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7059	/* no break */
7060	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7061	{
7062	switch (uVector)
7063	{
7064	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7065	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7066	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7067	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7068	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7069	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7070	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7071	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7072	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7073	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7074	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7075	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7076	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7077	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7078	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7079	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7080	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7081	#endif
7082	default:
7083	{
7084	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7085	AssertRCReturn(rc, rc);
7086
7087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7088	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7089	{
7090	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7091	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7092	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7093	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7094	AssertRCReturn(rc, rc);
7095	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7096	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7097	0 /* GCPtrFaultAddress */);
7098	AssertRCReturn(rc, rc);
7099	}
7100	else
7101	{
7102	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7103	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7104	}
7105	break;
7106	}
7107	}
7108	break;
7109	}
7110
7111	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7112	default:
7113	{
7114	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7115	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7116	break;
7117	}
7118	}
7119	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7120	return rc;
7121	}
7122
7123
7124	/**
7125	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7126	*/
7127	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7128	{
7129	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7130
7131	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7132	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7133	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7134	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7135	AssertRCReturn(rc, rc);
7136
7137	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7139	return VINF_SUCCESS;
7140	}
7141
7142
7143	/**
7144	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7145	*/
7146	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7147	{
7148	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7149	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7150	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7151	}
7152
7153
7154	/**
7155	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7156	*/
7157	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7158	{
7159	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7160	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7161	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7162	}
7163
7164
7165	/**
7166	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7167	*/
7168	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7169	{
7170	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7171	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7172	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7173	}
7174
7175
7176	/**
7177	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7178	*/
7179	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7180	{
7181	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7182	PVM pVM = pVCpu->CTX_SUFF(pVM);
7183	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7184	if (RT_LIKELY(rc == VINF_SUCCESS))
7185	{
7186	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7187	Assert(pVmxTransient->cbInstr == 2);
7188	}
7189	else
7190	{
7191	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7192	rc = VERR_EM_INTERPRETER;
7193	}
7194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7195	return rc;
7196	}
7197
7198
7199	/**
7200	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7201	*/
7202	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7203	{
7204	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7205	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7206	AssertRCReturn(rc, rc);
7207
7208	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7209	return VINF_EM_RAW_EMULATE_INSTR;
7210
7211	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7212	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7213	}
7214
7215
7216	/**
7217	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7218	*/
7219	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7220	{
7221	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7222	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7223	AssertRCReturn(rc, rc);
7224
7225	PVM pVM = pVCpu->CTX_SUFF(pVM);
7226	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7227	if (RT_LIKELY(rc == VINF_SUCCESS))
7228	{
7229	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7230	Assert(pVmxTransient->cbInstr == 2);
7231	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7232	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7233	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7234	}
7235	else
7236	{
7237	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7238	rc = VERR_EM_INTERPRETER;
7239	}
7240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7241	return rc;
7242	}
7243
7244
7245	/**
7246	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7247	*/
7248	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7249	{
7250	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7251	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7252	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7253	AssertRCReturn(rc, rc);
7254
7255	PVM pVM = pVCpu->CTX_SUFF(pVM);
7256	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7257	if (RT_LIKELY(rc == VINF_SUCCESS))
7258	{
7259	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7260	Assert(pVmxTransient->cbInstr == 3);
7261	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7262	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7263	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7264	}
7265	else
7266	{
7267	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7268	rc = VERR_EM_INTERPRETER;
7269	}
7270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7271	return rc;
7272	}
7273
7274
7275	/**
7276	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7277	*/
7278	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7279	{
7280	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7281	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7282	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7283	AssertRCReturn(rc, rc);
7284
7285	PVM pVM = pVCpu->CTX_SUFF(pVM);
7286	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7287	if (RT_LIKELY(rc == VINF_SUCCESS))
7288	{
7289	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7290	Assert(pVmxTransient->cbInstr == 2);
7291	}
7292	else
7293	{
7294	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7295	rc = VERR_EM_INTERPRETER;
7296	}
7297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7298	return rc;
7299	}
7300
7301
7302	/**
7303	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7304	*/
7305	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7306	{
7307	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7308	PVM pVM = pVCpu->CTX_SUFF(pVM);
7309	Assert(!pVM->hm.s.fNestedPaging);
7310
7311	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7312	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7313	AssertRCReturn(rc, rc);
7314
7315	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7316	rc = VBOXSTRICTRC_VAL(rc2);
7317	if (RT_LIKELY(rc == VINF_SUCCESS))
7318	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7319	else
7320	{
7321	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7322	pVmxTransient->uExitQualification, rc));
7323	}
7324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7325	return rc;
7326	}
7327
7328
7329	/**
7330	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7331	*/
7332	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7333	{
7334	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7335	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7336	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7337	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7338	AssertRCReturn(rc, rc);
7339
7340	PVM pVM = pVCpu->CTX_SUFF(pVM);
7341	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7342	if (RT_LIKELY(rc == VINF_SUCCESS))
7343	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7344	else
7345	{
7346	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7347	rc = VERR_EM_INTERPRETER;
7348	}
7349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7350	return rc;
7351	}
7352
7353
7354	/**
7355	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7356	*/
7357	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7358	{
7359	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7360	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7361	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7362	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7363	AssertRCReturn(rc, rc);
7364
7365	PVM pVM = pVCpu->CTX_SUFF(pVM);
7366	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7367	rc = VBOXSTRICTRC_VAL(rc2);
7368	if (RT_LIKELY( rc == VINF_SUCCESS
7369	\|\| rc == VINF_EM_HALT))
7370	{
7371	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7372	AssertRCReturn(rc3, rc3);
7373
7374	if ( rc == VINF_EM_HALT
7375	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7376	{
7377	rc = VINF_SUCCESS;
7378	}
7379	}
7380	else
7381	{
7382	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7383	rc = VERR_EM_INTERPRETER;
7384	}
7385	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7386	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7388	return rc;
7389	}
7390
7391
7392	/**
7393	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7394	*/
7395	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7396	{
7397	/*
7398	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7399	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7400	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7401	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7402	*/
7403	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7404	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7405	}
7406
7407
7408	/**
7409	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7410	*/
7411	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7412	{
7413	/*
7414	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7415	* root operation. If we get there there is something funny going on.
7416	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7417	*/
7418	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7419	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7420	}
7421
7422
7423	/**
7424	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7425	*/
7426	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7427	{
7428	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7429	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7430	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7431	}
7432
7433
7434	/**
7435	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7436	*/
7437	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7438	{
7439	/*
7440	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7441	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7442	* See Intel spec. 25.3 "Other Causes of VM-exits".
7443	*/
7444	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7445	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7446	}
7447
7448
7449	/**
7450	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7451	* VM-exit.
7452	*/
7453	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7454	{
7455	/*
7456	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7457	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7458	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7459	*/
7460	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7461	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7462	}
7463
7464
7465	/**
7466	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7467	* VM-exit.
7468	*/
7469	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7470	{
7471	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7472	return VINF_EM_RESET;
7473	}
7474
7475
7476	/**
7477	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7478	*/
7479	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7480	{
7481	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7482	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7483	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7484	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7485	AssertRCReturn(rc, rc);
7486
7487	pMixedCtx->rip++;
7488	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7489	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7490	rc = VINF_SUCCESS;
7491	else
7492	rc = VINF_EM_HALT;
7493
7494	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7495	return rc;
7496	}
7497
7498
7499	/**
7500	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7501	*/
7502	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7503	{
7504	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7505	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7506	return VINF_SUCCESS;
7507	}
7508
7509
7510	/**
7511	* VM-exit handler for expiry of the VMX preemption timer.
7512	*/
7513	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7514	{
7515	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7516
7517	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7518	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7519
7520	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7521	PVM pVM = pVCpu->CTX_SUFF(pVM);
7522	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7524	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7525	}
7526
7527
7528	/**
7529	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7530	*/
7531	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7532	{
7533	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7534	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7535	/** @todo check if XSETBV is supported by the recompiler. */
7536	return VERR_EM_INTERPRETER;
7537	}
7538
7539
7540	/**
7541	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7542	*/
7543	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7544	{
7545	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7546	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7547	/** @todo implement EMInterpretInvpcid() */
7548	return VERR_EM_INTERPRETER;
7549	}
7550
7551
7552	/**
7553	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7554	* Error VM-exit.
7555	*/
7556	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7557	{
7558	uint32_t uIntrState;
7559	HMVMXHCUINTREG uHCReg;
7560	uint64_t u64Val;
7561	uint32_t u32Val;
7562
7563	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7564	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7565	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7566	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7567	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7568	AssertRCReturn(rc, rc);
7569
7570	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7571	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7572	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7573	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7574
7575	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7576	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7577	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7578	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7579	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7580	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7581	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7582	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7583	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7584	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7585	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7586	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7587
7588	PVM pVM = pVCpu->CTX_SUFF(pVM);
7589	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7590
7591	return VERR_VMX_INVALID_GUEST_STATE;
7592	}
7593
7594
7595	/**
7596	* VM-exit handler for VM-entry failure due to an MSR-load
7597	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7598	*/
7599	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7600	{
7601	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7602	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7603	}
7604
7605
7606	/**
7607	* VM-exit handler for VM-entry failure due to a machine-check event
7608	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7609	*/
7610	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7611	{
7612	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7613	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7614	}
7615
7616
7617	/**
7618	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7619	* theory.
7620	*/
7621	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7622	{
7623	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7624	return VERR_VMX_UNDEFINED_EXIT_CODE;
7625	}
7626
7627
7628	/**
7629	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7630	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7631	* Conditional VM-exit.
7632	*/
7633	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7634	{
7635	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7636	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7637	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7638	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7639	return VERR_EM_INTERPRETER;
7640	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7641	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7642	}
7643
7644
7645	/**
7646	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7647	*/
7648	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7649	{
7650	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7651	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7652	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7653	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7654	return VERR_EM_INTERPRETER;
7655	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7656	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7657	}
7658
7659
7660	/**
7661	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7662	*/
7663	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7664	{
7665	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7666	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7667	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7668	AssertRCReturn(rc, rc);
7669	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7670	AssertRCReturn(rc, rc);
7671	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7672	AssertRCReturn(rc, rc);
7673
7674	PVM pVM = pVCpu->CTX_SUFF(pVM);
7675	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7676	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7677	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7679
7680	if (RT_LIKELY(rc == VINF_SUCCESS))
7681	{
7682	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7683	Assert(pVmxTransient->cbInstr == 2);
7684	}
7685	return rc;
7686	}
7687
7688
7689	/**
7690	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7691	*/
7692	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7693	{
7694	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7695	PVM pVM = pVCpu->CTX_SUFF(pVM);
7696	int rc = VINF_SUCCESS;
7697
7698	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7699	if ( pVM->hm.s.fTPRPatchingActive
7700	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7701	{
7702	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7703	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7704	{
7705	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7706	AssertRC(rc);
7707	}
7708
7709	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7710	Assert(pVmxTransient->cbInstr == 2);
7711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7712	return VINF_SUCCESS;
7713	}
7714
7715	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7716	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7717	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7718	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7719	AssertRCReturn(rc, rc);
7720	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7721
7722	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7723	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7724	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7725
7726	if (RT_LIKELY(rc == VINF_SUCCESS))
7727	{
7728	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7729
7730	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7731	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7732	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7733	{
7734	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7735	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7736	}
7737	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7738	{
7739	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7740	AssertRCReturn(rc, rc);
7741	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7742	}
7743	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7744	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7745
7746	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7747	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7748	{
7749	switch (pMixedCtx->ecx)
7750	{
7751	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7752	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7753	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7754	case MSR_K8_FS_BASE: /* no break */
7755	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7756	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7757	}
7758	}
7759	#ifdef VBOX_STRICT
7760	else
7761	{
7762	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7763	switch (pMixedCtx->ecx)
7764	{
7765	case MSR_IA32_SYSENTER_CS:
7766	case MSR_IA32_SYSENTER_EIP:
7767	case MSR_IA32_SYSENTER_ESP:
7768	case MSR_K8_FS_BASE:
7769	case MSR_K8_GS_BASE:
7770	{
7771	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7772	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7773	}
7774
7775	case MSR_K8_LSTAR:
7776	case MSR_K6_STAR:
7777	case MSR_K8_SF_MASK:
7778	case MSR_K8_TSC_AUX:
7779	case MSR_K8_KERNEL_GS_BASE:
7780	{
7781	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7782	pMixedCtx->ecx));
7783	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7784	}
7785	}
7786	}
7787	#endif /* VBOX_STRICT */
7788	}
7789	return rc;
7790	}
7791
7792
7793	/**
7794	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7795	*/
7796	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7797	{
7798	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7799	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7801	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7802	return VERR_EM_INTERPRETER;
7803	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7804	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7805	}
7806
7807
7808	/**
7809	* VM-exit handler for when the TPR value is lowered below the specified
7810	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7811	*/
7812	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7813	{
7814	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7815	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7816
7817	/*
7818	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7819	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7820	* resume guest execution.
7821	*/
7822	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7824	return VINF_SUCCESS;
7825	}
7826
7827
7828	/**
7829	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7830	* VM-exit.
7831	*
7832	* @retval VINF_SUCCESS when guest execution can continue.
7833	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7834	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7835	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7836	* recompiler.
7837	*/
7838	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7839	{
7840	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7841	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7842	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7843	AssertRCReturn(rc, rc);
7844
7845	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7846	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7847	PVM pVM = pVCpu->CTX_SUFF(pVM);
7848	switch (uAccessType)
7849	{
7850	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7851	{
7852	#if 0
7853	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7854	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7855	#else
7856	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7857	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7858	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7859	#endif
7860	AssertRCReturn(rc, rc);
7861
7862	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7863	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7864	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7865	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7866
7867	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7868	{
7869	case 0: /* CR0 */
7870	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7871	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7872	break;
7873	case 2: /* C2 **/
7874	/* Nothing to do here, CR2 it's not part of the VMCS. */
7875	break;
7876	case 3: /* CR3 */
7877	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7878	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7879	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7880	break;
7881	case 4: /* CR4 */
7882	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7883	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7884	break;
7885	case 8: /* CR8 */
7886	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7887	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7888	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7889	break;
7890	default:
7891	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7892	break;
7893	}
7894
7895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7896	break;
7897	}
7898
7899	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7900	{
7901	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7902	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7903	AssertRCReturn(rc, rc);
7904	Assert( !pVM->hm.s.fNestedPaging
7905	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7906	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7907
7908	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7909	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7910	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7911
7912	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7913	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7914	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7915	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7917	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7918	break;
7919	}
7920
7921	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7922	{
7923	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7924	AssertRCReturn(rc, rc);
7925	rc = EMInterpretCLTS(pVM, pVCpu);
7926	AssertRCReturn(rc, rc);
7927	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7928	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7929	Log(("CRX CLTS write rc=%d\n", rc));
7930	break;
7931	}
7932
7933	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7934	{
7935	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7936	AssertRCReturn(rc, rc);
7937	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7938	if (RT_LIKELY(rc == VINF_SUCCESS))
7939	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7941	Log(("CRX LMSW write rc=%d\n", rc));
7942	break;
7943	}
7944
7945	default:
7946	{
7947	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7948	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7949	}
7950	}
7951
7952	/* Validate possible error codes. */
7953	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7954	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7955	if (RT_SUCCESS(rc))
7956	{
7957	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7958	AssertRCReturn(rc2, rc2);
7959	}
7960
7961	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7962	return rc;
7963	}
7964
7965
7966	/**
7967	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7968	* VM-exit.
7969	*/
7970	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7971	{
7972	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7973	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7974
7975	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7976	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7977	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7978	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7979	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7980	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7981	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7982	AssertRCReturn(rc, rc);
7983	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7984
7985	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7986	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7987	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7988	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7989	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7990	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7991	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7992
7993	/* I/O operation lookup arrays. */
7994	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7995	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7996
7997	const uint32_t cbSize = s_aIOSize[uIOWidth];
7998	const uint32_t cbInstr = pVmxTransient->cbInstr;
7999	PVM pVM = pVCpu->CTX_SUFF(pVM);
8000	if (fIOString)
8001	{
8002	/* INS/OUTS - I/O String instruction. */
8003	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8004	/** @todo for now manually disassemble later optimize by getting the fields from
8005	* the VMCS. */
8006	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8007	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8008	* segment prefix info. */
8009	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8010	if (RT_SUCCESS(rc))
8011	{
8012	if (fIOWrite)
8013	{
8014	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8015	(DISCPUMODE)pDis->uAddrMode, cbSize);
8016	rc = VBOXSTRICTRC_VAL(rc2);
8017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8018	}
8019	else
8020	{
8021	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8022	(DISCPUMODE)pDis->uAddrMode, cbSize);
8023	rc = VBOXSTRICTRC_VAL(rc2);
8024	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8025	}
8026	}
8027	else
8028	{
8029	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8030	rc = VINF_EM_RAW_EMULATE_INSTR;
8031	}
8032	}
8033	else
8034	{
8035	/* IN/OUT - I/O instruction. */
8036	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8037	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8038	if (fIOWrite)
8039	{
8040	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8041	rc = VBOXSTRICTRC_VAL(rc2);
8042	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8043	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8044	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8045	}
8046	else
8047	{
8048	uint32_t u32Result = 0;
8049	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8050	rc = VBOXSTRICTRC_VAL(rc2);
8051	if (IOM_SUCCESS(rc))
8052	{
8053	/* Save result of I/O IN instr. in AL/AX/EAX. */
8054	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8055	}
8056	else if (rc == VINF_IOM_R3_IOPORT_READ)
8057	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8059	}
8060	}
8061
8062	if (IOM_SUCCESS(rc))
8063	{
8064	pMixedCtx->rip += cbInstr;
8065	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8066	if (RT_LIKELY(rc == VINF_SUCCESS))
8067	{
8068	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8069	AssertRCReturn(rc, rc);
8070
8071	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8072	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8073	{
8074	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8075	for (unsigned i = 0; i < 4; i++)
8076	{
8077	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8078	if ( ( uIOPort >= pMixedCtx->dr[i]
8079	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8080	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8081	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8082	{
8083	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8084	uint64_t uDR6 = ASMGetDR6();
8085
8086	/* Clear all breakpoint status flags and set the one we just hit. */
8087	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8088	uDR6 \|= (uint64_t)RT_BIT(i);
8089
8090	/*
8091	* Note: AMD64 Architecture Programmer's Manual 13.1:
8092	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8093	* be cleared by software after the contents have been read.
8094	*/
8095	ASMSetDR6(uDR6);
8096
8097	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8098	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8099
8100	/* Paranoia. */
8101	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8102	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8103	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8104
8105	/* Resync DR7 */
8106	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8107	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8108
8109	/* Set #DB to be injected into the VM and continue guest execution. */
8110	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8111	break;
8112	}
8113	}
8114	}
8115	}
8116	}
8117
8118	#ifdef DEBUG
8119	if (rc == VINF_IOM_R3_IOPORT_READ)
8120	Assert(!fIOWrite);
8121	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8122	Assert(fIOWrite);
8123	else
8124	{
8125	AssertMsg( RT_FAILURE(rc)
8126	\|\| rc == VINF_SUCCESS
8127	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8128	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8129	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8130	}
8131	#endif
8132
8133	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8134	return rc;
8135	}
8136
8137
8138	/**
8139	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8140	* VM-exit.
8141	*/
8142	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8143	{
8144	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8145
8146	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8147	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8148	AssertRCReturn(rc, rc);
8149	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8150	{
8151	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8152	AssertRCReturn(rc, rc);
8153	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8154	{
8155	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8156	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8157	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8158	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8159	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8160	{
8161	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8162	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8163
8164	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8165	Assert(!pVCpu->hm.s.Event.fPending);
8166	pVCpu->hm.s.Event.fPending = true;
8167	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8168	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8169	AssertRCReturn(rc, rc);
8170	if (fErrorCodeValid)
8171	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8172	else
8173	pVCpu->hm.s.Event.u32ErrCode = 0;
8174	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8175	&& uVector == X86_XCPT_PF)
8176	{
8177	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8178	}
8179	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8180	}
8181	}
8182	}
8183	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8184	* emulation. */
8185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8186	return VERR_EM_INTERPRETER;
8187	}
8188
8189
8190	/**
8191	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8192	*/
8193	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8194	{
8195	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8196	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8197	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8198	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8199	AssertRCReturn(rc, rc);
8200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8201	return VINF_EM_DBG_STOP;
8202	}
8203
8204
8205	/**
8206	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8207	*/
8208	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8209	{
8210	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8211
8212	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8213	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8214	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8215	return VINF_SUCCESS;
8216	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8217	return rc;
8218
8219	#if 0
8220	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8221	* just sync the whole thing. */
8222	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8223	#else
8224	/* Aggressive state sync. for now. */
8225	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8226	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8227	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8228	#endif
8229	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8230	AssertRCReturn(rc, rc);
8231
8232	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8233	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8234	switch (uAccessType)
8235	{
8236	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8237	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8238	{
8239	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8240	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8241	{
8242	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8243	}
8244
8245	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8246	GCPhys &= PAGE_BASE_GC_MASK;
8247	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8248	PVM pVM = pVCpu->CTX_SUFF(pVM);
8249	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8250	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8251
8252	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8253	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8254	CPUMCTX2CORE(pMixedCtx), GCPhys);
8255	rc = VBOXSTRICTRC_VAL(rc2);
8256	Log(("ApicAccess rc=%d\n", rc));
8257	if ( rc == VINF_SUCCESS
8258	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8259	\|\| rc == VERR_PAGE_NOT_PRESENT)
8260	{
8261	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8262	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8263	rc = VINF_SUCCESS;
8264	}
8265	break;
8266	}
8267
8268	default:
8269	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8270	rc = VINF_EM_RAW_EMULATE_INSTR;
8271	break;
8272	}
8273
8274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8275	return rc;
8276	}
8277
8278
8279	/**
8280	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8281	* VM-exit.
8282	*/
8283	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8284	{
8285	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8286
8287	/* We should -not- get this VM-exit if the guest is debugging. */
8288	if (CPUMIsGuestDebugStateActive(pVCpu))
8289	{
8290	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8291	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8292	}
8293
8294	int rc = VERR_INTERNAL_ERROR_5;
8295	if ( !DBGFIsStepping(pVCpu)
8296	&& !CPUMIsHyperDebugStateActive(pVCpu))
8297	{
8298	/* Don't intercept MOV DRx. */
8299	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8300	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8301	AssertRCReturn(rc, rc);
8302
8303	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8304	PVM pVM = pVCpu->CTX_SUFF(pVM);
8305	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8306	AssertRC(rc);
8307	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8308
8309	#ifdef VBOX_WITH_STATISTICS
8310	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8311	AssertRCReturn(rc, rc);
8312	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8314	else
8315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8316	#endif
8317	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8318	return VINF_SUCCESS;
8319	}
8320
8321	/*
8322	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8323	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8324	*/
8325	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8326	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8327	AssertRCReturn(rc, rc);
8328
8329	PVM pVM = pVCpu->CTX_SUFF(pVM);
8330	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8331	{
8332	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8333	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8334	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8335	if (RT_SUCCESS(rc))
8336	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8338	}
8339	else
8340	{
8341	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8342	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8343	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8344	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8345	}
8346
8347	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8348	if (RT_SUCCESS(rc))
8349	{
8350	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8351	AssertRCReturn(rc2, rc2);
8352	}
8353	return rc;
8354	}
8355
8356
8357	/**
8358	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8359	* Conditional VM-exit.
8360	*/
8361	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8362	{
8363	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8364	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8365
8366	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8367	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8368	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8369	return VINF_SUCCESS;
8370	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8371	return rc;
8372
8373	RTGCPHYS GCPhys = 0;
8374	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8375
8376	#if 0
8377	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8378	#else
8379	/* Aggressive state sync. for now. */
8380	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8381	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8382	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8383	#endif
8384	AssertRCReturn(rc, rc);
8385
8386	/*
8387	* If we succeed, resume guest execution.
8388	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8389	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8390	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8391	* weird case. See @bugref{6043}.
8392	*/
8393	PVM pVM = pVCpu->CTX_SUFF(pVM);
8394	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8395	rc = VBOXSTRICTRC_VAL(rc2);
8396	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8397	if ( rc == VINF_SUCCESS
8398	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8399	\|\| rc == VERR_PAGE_NOT_PRESENT)
8400	{
8401	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8402	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8403	return VINF_SUCCESS;
8404	}
8405	return rc;
8406	}
8407
8408
8409	/**
8410	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8411	* VM-exit.
8412	*/
8413	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8414	{
8415	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8416	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8417
8418	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8419	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8420	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8421	return VINF_SUCCESS;
8422	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8423	return rc;
8424
8425	RTGCPHYS GCPhys = 0;
8426	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8427	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8428	#if 0
8429	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8430	#else
8431	/* Aggressive state sync. for now. */
8432	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8433	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8434	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8435	#endif
8436	AssertRCReturn(rc, rc);
8437
8438	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8439	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8440
8441	RTGCUINT uErrorCode = 0;
8442	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8443	uErrorCode \|= X86_TRAP_PF_ID;
8444	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8445	uErrorCode \|= X86_TRAP_PF_RW;
8446	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8447	uErrorCode \|= X86_TRAP_PF_P;
8448
8449	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8450
8451	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8452	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8453
8454	/* Handle the pagefault trap for the nested shadow table. */
8455	PVM pVM = pVCpu->CTX_SUFF(pVM);
8456	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8457	TRPMResetTrap(pVCpu);
8458
8459	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8460	if ( rc == VINF_SUCCESS
8461	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8462	\|\| rc == VERR_PAGE_NOT_PRESENT)
8463	{
8464	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8465	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8466	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8467	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8468	return VINF_SUCCESS;
8469	}
8470
8471	Log(("EPT return to ring-3 rc=%d\n"));
8472	return rc;
8473	}
8474
8475
8476	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8477	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8478	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8479	/**
8480	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8481	*/
8482	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8483	{
8484	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8486
8487	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8488	AssertRCReturn(rc, rc);
8489
8490	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8491	{
8492	/* Old-style FPU error reporting needs some extra work. */
8493	/** @todo don't fall back to the recompiler, but do it manually. */
8494	return VERR_EM_INTERPRETER;
8495	}
8496	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8497	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8498	return rc;
8499	}
8500
8501
8502	/**
8503	* VM-exit exception handler for #BP (Breakpoint exception).
8504	*/
8505	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8506	{
8507	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8509
8510	/** @todo Try optimize this by not saving the entire guest state unless
8511	* really needed. */
8512	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8513	AssertRCReturn(rc, rc);
8514
8515	PVM pVM = pVCpu->CTX_SUFF(pVM);
8516	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8517	if (rc == VINF_EM_RAW_GUEST_TRAP)
8518	{
8519	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8520	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8521	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8522	AssertRCReturn(rc, rc);
8523
8524	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8525	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8526	}
8527
8528	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8529	return rc;
8530	}
8531
8532
8533	/**
8534	* VM-exit exception handler for #DB (Debug exception).
8535	*/
8536	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8537	{
8538	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8540
8541	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8542	AssertRCReturn(rc, rc);
8543
8544	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8545	uint64_t uDR6 = X86_DR6_INIT_VAL;
8546	uDR6 \|= (pVmxTransient->uExitQualification
8547	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8548	PVM pVM = pVCpu->CTX_SUFF(pVM);
8549	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8550	if (rc == VINF_EM_RAW_GUEST_TRAP)
8551	{
8552	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8553	pMixedCtx->dr[6] = uDR6;
8554
8555	if (CPUMIsGuestDebugStateActive(pVCpu))
8556	ASMSetDR6(pMixedCtx->dr[6]);
8557
8558	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8559
8560	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8561	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8562
8563	/* Paranoia. */
8564	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8565	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8566	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8567
8568	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8569	AssertRCReturn(rc,rc);
8570
8571	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8572	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8573	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8574	AssertRCReturn(rc2, rc2);
8575	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8576	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8577	rc = VINF_SUCCESS;
8578	}
8579
8580	return rc;
8581	}
8582
8583
8584	/**
8585	* VM-exit exception handler for #NM (Device-not-available exception: floating
8586	* point exception).
8587	*/
8588	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8589	{
8590	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8591
8592	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8593	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8594	#endif
8595
8596	/* We require CR0 and EFER. EFER is always up-to-date. */
8597	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8598	AssertRCReturn(rc, rc);
8599
8600	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8601	PVM pVM = pVCpu->CTX_SUFF(pVM);
8602	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8603	if (rc == VINF_SUCCESS)
8604	{
8605	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8606	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8608	return VINF_SUCCESS;
8609	}
8610
8611	/* Forward #NM to the guest. */
8612	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8613	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8614	AssertRCReturn(rc, rc);
8615	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8616	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8617	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8618	return rc;
8619	}
8620
8621
8622	/**
8623	* VM-exit exception handler for #GP (General-protection exception).
8624	*
8625	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8626	*/
8627	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8628	{
8629	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8631
8632	int rc = VERR_INTERNAL_ERROR_5;
8633	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8634	{
8635	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8636	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8637	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8638	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8639	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8640	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8641	AssertRCReturn(rc, rc);
8642	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8643	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8644	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8645	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8646	return rc;
8647	#else
8648	/* We don't intercept #GP. */
8649	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8650	return VERR_VMX_UNEXPECTED_EXCEPTION;
8651	#endif
8652	}
8653
8654	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8655	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8656
8657	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8658	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8659	AssertRCReturn(rc, rc);
8660
8661	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8662	uint32_t cbOp = 0;
8663	PVM pVM = pVCpu->CTX_SUFF(pVM);
8664	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8665	if (RT_SUCCESS(rc))
8666	{
8667	rc = VINF_SUCCESS;
8668	Assert(cbOp == pDis->cbInstr);
8669	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8670	switch (pDis->pCurInstr->uOpcode)
8671	{
8672	case OP_CLI:
8673	pMixedCtx->eflags.Bits.u1IF = 0;
8674	pMixedCtx->rip += pDis->cbInstr;
8675	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8676	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8677	break;
8678
8679	case OP_STI:
8680	pMixedCtx->eflags.Bits.u1IF = 1;
8681	pMixedCtx->rip += pDis->cbInstr;
8682	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8683	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8684	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8686	break;
8687
8688	case OP_HLT:
8689	rc = VINF_EM_HALT;
8690	pMixedCtx->rip += pDis->cbInstr;
8691	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8692	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8693	break;
8694
8695	case OP_POPF:
8696	{
8697	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8698	uint32_t cbParm = 0;
8699	uint32_t uMask = 0;
8700	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8701	{
8702	cbParm = 4;
8703	uMask = 0xffffffff;
8704	}
8705	else
8706	{
8707	cbParm = 2;
8708	uMask = 0xffff;
8709	}
8710
8711	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8712	RTGCPTR GCPtrStack = 0;
8713	X86EFLAGS uEflags;
8714	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8715	&GCPtrStack);
8716	if (RT_SUCCESS(rc))
8717	{
8718	Assert(sizeof(uEflags.u32) >= cbParm);
8719	uEflags.u32 = 0;
8720	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8721	}
8722	if (RT_FAILURE(rc))
8723	{
8724	rc = VERR_EM_INTERPRETER;
8725	break;
8726	}
8727	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8728	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8729	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8730	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8731	pMixedCtx->eflags.Bits.u1RF = 0;
8732	pMixedCtx->esp += cbParm;
8733	pMixedCtx->esp &= uMask;
8734	pMixedCtx->rip += pDis->cbInstr;
8735	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8736	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8737	break;
8738	}
8739
8740	case OP_PUSHF:
8741	{
8742	uint32_t cbParm = 0;
8743	uint32_t uMask = 0;
8744	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8745	{
8746	cbParm = 4;
8747	uMask = 0xffffffff;
8748	}
8749	else
8750	{
8751	cbParm = 2;
8752	uMask = 0xffff;
8753	}
8754
8755	/* Get the stack pointer & push the contents of eflags onto the stack. */
8756	RTGCPTR GCPtrStack = 0;
8757	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8758	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8759	if (RT_FAILURE(rc))
8760	{
8761	rc = VERR_EM_INTERPRETER;
8762	break;
8763	}
8764	X86EFLAGS uEflags;
8765	uEflags = pMixedCtx->eflags;
8766	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8767	uEflags.Bits.u1RF = 0;
8768	uEflags.Bits.u1VM = 0;
8769
8770	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8771	if (RT_FAILURE(rc))
8772	{
8773	rc = VERR_EM_INTERPRETER;
8774	break;
8775	}
8776	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8777	pMixedCtx->esp -= cbParm;
8778	pMixedCtx->esp &= uMask;
8779	pMixedCtx->rip += pDis->cbInstr;
8780	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8782	break;
8783	}
8784
8785	case OP_IRET:
8786	{
8787	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8788	* instruction reference. */
8789	RTGCPTR GCPtrStack = 0;
8790	uint32_t uMask = 0xffff;
8791	uint16_t aIretFrame[3];
8792	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8793	{
8794	rc = VERR_EM_INTERPRETER;
8795	break;
8796	}
8797	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8798	&GCPtrStack);
8799	if (RT_SUCCESS(rc))
8800	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8801	if (RT_FAILURE(rc))
8802	{
8803	rc = VERR_EM_INTERPRETER;
8804	break;
8805	}
8806	pMixedCtx->eip = 0;
8807	pMixedCtx->ip = aIretFrame[0];
8808	pMixedCtx->cs.Sel = aIretFrame[1];
8809	pMixedCtx->cs.ValidSel = aIretFrame[1];
8810	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8811	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8812	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8813	pMixedCtx->sp += sizeof(aIretFrame);
8814	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8815	\| HM_CHANGED_GUEST_RFLAGS;
8816	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8818	break;
8819	}
8820
8821	case OP_INT:
8822	{
8823	uint16_t uVector = pDis->Param1.uValue & 0xff;
8824	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8825	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8826	break;
8827	}
8828
8829	case OP_INTO:
8830	{
8831	if (pMixedCtx->eflags.Bits.u1OF)
8832	{
8833	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8834	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8835	}
8836	break;
8837	}
8838
8839	default:
8840	{
8841	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8842	EMCODETYPE_SUPERVISOR);
8843	rc = VBOXSTRICTRC_VAL(rc2);
8844	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8845	Log(("#GP rc=%Rrc\n", rc));
8846	break;
8847	}
8848	}
8849	}
8850	else
8851	rc = VERR_EM_INTERPRETER;
8852
8853	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8854	("#GP Unexpected rc=%Rrc\n", rc));
8855	return rc;
8856	}
8857
8858
8859	/**
8860	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8861	* the exception reported in the VMX transient structure back into the VM.
8862	*
8863	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8864	* up-to-date.
8865	*/
8866	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8867	{
8868	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8869
8870	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8871	hmR0VmxCheckExitDueToEventDelivery(). */
8872	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8873	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8874	AssertRCReturn(rc, rc);
8875	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8876	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8877	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8878	return VINF_SUCCESS;
8879	}
8880
8881
8882	/**
8883	* VM-exit exception handler for #PF (Page-fault exception).
8884	*/
8885	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8886	{
8887	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8888	PVM pVM = pVCpu->CTX_SUFF(pVM);
8889	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8890	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8891	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8892	AssertRCReturn(rc, rc);
8893
8894	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8895	if (pVM->hm.s.fNestedPaging)
8896	{
8897	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8898	{
8899	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8900	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8901	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8902	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8904	}
8905	else
8906	{
8907	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8908	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8909	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8910	Log(("Pending #DF due to vectoring #PF. NP\n"));
8911	}
8912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8913	return rc;
8914	}
8915	#else
8916	Assert(!pVM->hm.s.fNestedPaging);
8917	#endif
8918
8919	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8920	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8921	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8922	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8923	AssertRCReturn(rc, rc);
8924	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8925	if ( pVM->hm.s.fTRPPatchingAllowed
8926	&& pVM->hm.s.pGuestPatchMem
8927	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8928	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8929	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8930	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8931	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8932	{
8933	RTGCPHYS GCPhys;
8934	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8935	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8936	if ( rc == VINF_SUCCESS
8937	&& GCPhys == GCPhysApicBase)
8938	{
8939	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8940	AssertRCReturn(rc, rc);
8941
8942	/* Only attempt to patch the instruction once. */
8943	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8944	if (!pPatch)
8945	return VINF_EM_HM_PATCH_TPR_INSTR;
8946	}
8947	}
8948	#endif
8949
8950	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8951	AssertRCReturn(rc, rc);
8952
8953	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8954	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8955
8956	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8957	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8958	(RTGCPTR)pVmxTransient->uExitQualification);
8959
8960	Log(("#PF: rc=%Rrc\n", rc));
8961	if (rc == VINF_SUCCESS)
8962	{
8963	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8964	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8965	* memory? We don't update the whole state here... */
8966	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8967	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8968	TRPMResetTrap(pVCpu);
8969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8970	return rc;
8971	}
8972	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8973	{
8974	if (!pVmxTransient->fVectoringPF)
8975	{
8976	/* It's a guest page fault and needs to be reflected to the guest. */
8977	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8978	TRPMResetTrap(pVCpu);
8979	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8980	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8981	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8982	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8983	}
8984	else
8985	{
8986	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8987	TRPMResetTrap(pVCpu);
8988	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8989	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8990	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8991	}
8992
8993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8994	return VINF_SUCCESS;
8995	}
8996
8997	TRPMResetTrap(pVCpu);
8998	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8999	return rc;
9000	}
9001

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45918

Download in other formats: