HMVMXR0.cpp@ 46443

Last change on this file since 46443 was 46442, checked in by vboxsync, 12 years ago
VMM: HM bits, add changed-flag for CR2 as AMD-V has CR2 caching.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 387.1 KB

Line
1	/* $Id: HMVMXR0.cpp 46442 2013-06-07 14:34:39Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/**
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG uEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
174	uint64_t u64LStarMsr;
175	/** The guest's TPR value used for TPR shadowing. */
176	uint8_t u8GuestTpr;
177	/** Alignment. */
178	uint8_t abAlignment0[6];
179
180	/** The basic VM-exit reason. */
181	uint16_t uExitReason;
182	/** Alignment. */
183	uint16_t u16Alignment0;
184	/** The VM-exit interruption error code. */
185	uint32_t uExitIntrErrorCode;
186	/** The VM-exit exit qualification. */
187	uint64_t uExitQualification;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntrInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** Whether the VM-entry failed or not. */
194	bool fVMEntryFailed;
195	/** Alignment. */
196	uint8_t abAlignment1[5];
197
198	/** The VM-entry interruption-information field. */
199	uint32_t uEntryIntrInfo;
200	/** The VM-entry exception error code field. */
201	uint32_t uEntryXcptErrorCode;
202	/** The VM-entry instruction length field. */
203	uint32_t cbEntryInstr;
204
205	/** IDT-vectoring information field. */
206	uint32_t uIdtVectoringInfo;
207	/** IDT-vectoring error code. */
208	uint32_t uIdtVectoringErrorCode;
209
210	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
211	uint32_t fVmcsFieldsRead;
212	/** Whether TSC-offsetting should be setup before VM-entry. */
213	bool fUpdateTscOffsettingAndPreemptTimer;
214	/** Whether the VM-exit was caused by a page-fault during delivery of a
215	* contributary exception or a page-fault. */
216	bool fVectoringPF;
217	} VMXTRANSIENT, *PVMXTRANSIENT;
218	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
219	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
220	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
221
222
223	/**
224	* MSR-bitmap read permissions.
225	*/
226	typedef enum VMXMSREXITREAD
227	{
228	/** Reading this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_READ = 0xb,
230	/** Reading this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_READ
232	} VMXMSREXITREAD;
233
234	/**
235	* MSR-bitmap write permissions.
236	*/
237	typedef enum VMXMSREXITWRITE
238	{
239	/** Writing to this MSR causes a VM-exit. */
240	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
241	/** Writing to this MSR does not cause a VM-exit. */
242	VMXMSREXIT_PASSTHRU_WRITE
243	} VMXMSREXITWRITE;
244
245
246	/*******************************************************************************
247	* Internal Functions *
248	*******************************************************************************/
249	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
250	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
251	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
252	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
253	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
254	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
255	#endif
256	#ifndef HMVMX_USE_FUNCTION_TABLE
257	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
258	#define HMVMX_EXIT_DECL static int
259	#else
260	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
261	#endif
262
263	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307
308	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315
316
317	/*******************************************************************************
318	* Global Variables *
319	*******************************************************************************/
320	#ifdef HMVMX_USE_FUNCTION_TABLE
321	/**
322	* VM-exit handler.
323	*
324	* @returns VBox status code.
325	* @param pVCpu Pointer to the VMCPU.
326	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
327	* out-of-sync. Make sure to update the required
328	* fields before using them.
329	* @param pVmxTransient Pointer to the VMX-transient structure.
330	*/
331	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332	/** Pointer to VM-exit handler. */
333	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
334
335	/**
336	* VMX_EXIT dispatch table.
337	*/
338	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
339	{
340	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
341	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
342	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
343	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
344	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
345	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
346	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
347	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
348	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
349	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
350	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
351	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
352	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
353	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
354	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
355	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
356	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
357	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
358	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
359	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
360	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
361	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
362	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
363	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
364	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
365	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
366	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
367	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
368	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
369	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
370	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
371	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
372	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
373	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
374	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
375	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
376	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
377	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
378	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
379	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
380	/* 40 UNDEFINED */ hmR0VmxExitPause,
381	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
382	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
383	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
384	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
385	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
387	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
388	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
389	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
390	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
391	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
392	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
393	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
394	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
395	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
396	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
397	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
398	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
399	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
400	};
401	#endif /* HMVMX_USE_FUNCTION_TABLE */
402
403	#ifdef VBOX_STRICT
404	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
405	{
406	/* 0 */ "(Not Used)",
407	/* 1 */ "VMCALL executed in VMX root operation.",
408	/* 2 */ "VMCLEAR with invalid physical address.",
409	/* 3 */ "VMCLEAR with VMXON pointer.",
410	/* 4 */ "VMLAUNCH with non-clear VMCS.",
411	/* 5 */ "VMRESUME with non-launched VMCS.",
412	/* 6 */ "VMRESUME after VMXOFF",
413	/* 7 */ "VM entry with invalid control fields.",
414	/* 8 */ "VM entry with invalid host state fields.",
415	/* 9 */ "VMPTRLD with invalid physical address.",
416	/* 10 */ "VMPTRLD with VMXON pointer.",
417	/* 11 */ "VMPTRLD with incorrect revision identifier.",
418	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
419	/* 13 */ "VMWRITE to read-only VMCS component.",
420	/* 14 */ "(Not Used)",
421	/* 15 */ "VMXON executed in VMX root operation.",
422	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
423	/* 17 */ "VM entry with non-launched executing VMCS.",
424	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
425	/* 19 */ "VMCALL with non-clear VMCS.",
426	/* 20 */ "VMCALL with invalid VM-exit control fields.",
427	/* 21 */ "(Not Used)",
428	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
429	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
430	/* 24 */ "VMCALL with invalid SMM-monitor features.",
431	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
432	/* 26 */ "VM entry with events blocked by MOV SS.",
433	/* 27 */ "(Not Used)",
434	/* 28 */ "Invalid operand to INVEPT/INVVPID."
435	};
436	#endif /* VBOX_STRICT */
437
438
439
440	/**
441	* Updates the VM's last error record. If there was a VMX instruction error,
442	* reads the error data from the VMCS and updates VCPU's last error record as
443	* well.
444	*
445	* @param pVM Pointer to the VM.
446	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
447	* VERR_VMX_UNABLE_TO_START_VM or
448	* VERR_VMX_INVALID_VMCS_FIELD).
449	* @param rc The error code.
450	*/
451	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
452	{
453	AssertPtr(pVM);
454	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
455	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
456	{
457	AssertPtrReturnVoid(pVCpu);
458	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
459	}
460	pVM->hm.s.lLastError = rc;
461	}
462
463
464	/**
465	* Reads the VM-entry interruption-information field from the VMCS into the VMX
466	* transient structure.
467	*
468	* @returns VBox status code.
469	* @param pVmxTransient Pointer to the VMX transient structure.
470	*
471	* @remarks No-long-jump zone!!!
472	*/
473	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
474	{
475	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
476	AssertRCReturn(rc, rc);
477	return VINF_SUCCESS;
478	}
479
480
481	/**
482	* Reads the VM-entry exception error code field from the VMCS into
483	* the VMX transient structure.
484	*
485	* @returns VBox status code.
486	* @param pVmxTransient Pointer to the VMX transient structure.
487	*
488	* @remarks No-long-jump zone!!!
489	*/
490	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
491	{
492	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
493	AssertRCReturn(rc, rc);
494	return VINF_SUCCESS;
495	}
496
497
498	/**
499	* Reads the VM-entry exception error code field from the VMCS into
500	* the VMX transient structure.
501	*
502	* @returns VBox status code.
503	* @param pVCpu Pointer to the VMCPU.
504	* @param pVmxTransient Pointer to the VMX transient structure.
505	*
506	* @remarks No-long-jump zone!!!
507	*/
508	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
509	{
510	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
511	AssertRCReturn(rc, rc);
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Reads the VM-exit interruption-information field from the VMCS into the VMX
518	* transient structure.
519	*
520	* @returns VBox status code.
521	* @param pVCpu Pointer to the VMCPU.
522	* @param pVmxTransient Pointer to the VMX transient structure.
523	*/
524	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
525	{
526	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
527	{
528	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
529	AssertRCReturn(rc, rc);
530	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
531	}
532	return VINF_SUCCESS;
533	}
534
535
536	/**
537	* Reads the VM-exit interruption error code from the VMCS into the VMX
538	* transient structure.
539	*
540	* @returns VBox status code.
541	* @param pVCpu Pointer to the VMCPU.
542	* @param pVmxTransient Pointer to the VMX transient structure.
543	*/
544	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
545	{
546	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
547	{
548	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
549	AssertRCReturn(rc, rc);
550	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
551	}
552	return VINF_SUCCESS;
553	}
554
555
556	/**
557	* Reads the VM-exit instruction length field from the VMCS into the VMX
558	* transient structure.
559	*
560	* @returns VBox status code.
561	* @param pVCpu Pointer to the VMCPU.
562	* @param pVmxTransient Pointer to the VMX transient structure.
563	*/
564	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
565	{
566	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
567	{
568	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
569	AssertRCReturn(rc, rc);
570	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
571	}
572	return VINF_SUCCESS;
573	}
574
575
576	/**
577	* Reads the exit qualification from the VMCS into the VMX transient structure.
578	*
579	* @returns VBox status code.
580	* @param pVCpu Pointer to the VMCPU.
581	* @param pVmxTransient Pointer to the VMX transient structure.
582	*/
583	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
584	{
585	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
586	{
587	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
588	AssertRCReturn(rc, rc);
589	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
590	}
591	return VINF_SUCCESS;
592	}
593
594
595	/**
596	* Reads the IDT-vectoring information field from the VMCS into the VMX
597	* transient structure.
598	*
599	* @returns VBox status code.
600	* @param pVmxTransient Pointer to the VMX transient structure.
601	*
602	* @remarks No-long-jump zone!!!
603	*/
604	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
605	{
606	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
607	{
608	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
609	AssertRCReturn(rc, rc);
610	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
611	}
612	return VINF_SUCCESS;
613	}
614
615
616	/**
617	* Reads the IDT-vectoring error code from the VMCS into the VMX
618	* transient structure.
619	*
620	* @returns VBox status code.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Enters VMX root mode operation on the current CPU.
637	*
638	* @returns VBox status code.
639	* @param pVM Pointer to the VM (optional, can be NULL, after
640	* a resume).
641	* @param HCPhysCpuPage Physical address of the VMXON region.
642	* @param pvCpuPage Pointer to the VMXON region.
643	*/
644	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
645	{
646	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
647	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
648	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
649
650	if (pVM)
651	{
652	/* Write the VMCS revision dword to the VMXON region. */
653	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
654	}
655
656	/* Enable the VMX bit in CR4 if necessary. */
657	RTCCUINTREG uCr4 = ASMGetCR4();
658	if (!(uCr4 & X86_CR4_VMXE))
659	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
660
661	/* Enter VMX root mode. */
662	int rc = VMXEnable(HCPhysCpuPage);
663	if (RT_FAILURE(rc))
664	ASMSetCR4(uCr4);
665
666	return rc;
667	}
668
669
670	/**
671	* Exits VMX root mode operation on the current CPU.
672	*
673	* @returns VBox status code.
674	*/
675	static int hmR0VmxLeaveRootMode(void)
676	{
677	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
678
679	/* If we're for some reason not in VMX root mode, then don't leave it. */
680	RTCCUINTREG uHostCR4 = ASMGetCR4();
681	if (uHostCR4 & X86_CR4_VMXE)
682	{
683	/* Exit VMX root mode and clear the VMX bit in CR4. */
684	VMXDisable();
685	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
686	return VINF_SUCCESS;
687	}
688
689	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
690	}
691
692
693	/**
694	* Allocates and maps one physically contiguous page. The allocated page is
695	* zero'd out. (Used by various VT-x structures).
696	*
697	* @returns IPRT status code.
698	* @param pMemObj Pointer to the ring-0 memory object.
699	* @param ppVirt Where to store the virtual address of the
700	* allocation.
701	* @param pPhys Where to store the physical address of the
702	* allocation.
703	*/
704	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
705	{
706	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
707	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
708	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
709
710	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
711	if (RT_FAILURE(rc))
712	return rc;
713	ppVirt = RTR0MemObjAddress(pMemObj);
714	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
715	ASMMemZero32(*ppVirt, PAGE_SIZE);
716	return VINF_SUCCESS;
717	}
718
719
720	/**
721	* Frees and unmaps an allocated physical page.
722	*
723	* @param pMemObj Pointer to the ring-0 memory object.
724	* @param ppVirt Where to re-initialize the virtual address of
725	* allocation as 0.
726	* @param pHCPhys Where to re-initialize the physical address of the
727	* allocation as 0.
728	*/
729	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
730	{
731	AssertPtr(pMemObj);
732	AssertPtr(ppVirt);
733	AssertPtr(pHCPhys);
734	if (*pMemObj != NIL_RTR0MEMOBJ)
735	{
736	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
737	AssertRC(rc);
738	*pMemObj = NIL_RTR0MEMOBJ;
739	*ppVirt = 0;
740	*pHCPhys = 0;
741	}
742	}
743
744
745	/**
746	* Worker function to free VT-x related structures.
747	*
748	* @returns IPRT status code.
749	* @param pVM Pointer to the VM.
750	*/
751	static void hmR0VmxStructsFree(PVM pVM)
752	{
753	for (VMCPUID i = 0; i < pVM->cCpus; i++)
754	{
755	PVMCPU pVCpu = &pVM->aCpus[i];
756	AssertPtr(pVCpu);
757
758	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
759	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
761	#endif
762
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
765
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
768	}
769
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
772	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
773	#endif
774	}
775
776
777	/**
778	* Worker function to allocate VT-x related VM structures.
779	*
780	* @returns IPRT status code.
781	* @param pVM Pointer to the VM.
782	*/
783	static int hmR0VmxStructsAlloc(PVM pVM)
784	{
785	/*
786	* Initialize members up-front so we can cleanup properly on allocation failure.
787	*/
788	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
789	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
790	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
791	pVM->hm.s.vmx.HCPhys##a_Name = 0;
792
793	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
794	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
795	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
796	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
797
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
800	#endif
801	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
802
803	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
804	for (VMCPUID i = 0; i < pVM->cCpus; i++)
805	{
806	PVMCPU pVCpu = &pVM->aCpus[i];
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
813	#endif
814	}
815	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
816	#undef VMXLOCAL_INIT_VM_MEMOBJ
817
818	/*
819	* Allocate all the VT-x structures.
820	*/
821	int rc = VINF_SUCCESS;
822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
823	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
827	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
828	#endif
829
830	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
831	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
832	{
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
834	&pVM->hm.s.vmx.HCPhysApicAccess);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	/*
840	* Initialize per-VCPU VT-x structures.
841	*/
842	for (VMCPUID i = 0; i < pVM->cCpus; i++)
843	{
844	PVMCPU pVCpu = &pVM->aCpus[i];
845	AssertPtr(pVCpu);
846
847	/* Allocate the VM control structure (VMCS). */
848	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
854	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
855	{
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
857	&pVCpu->hm.s.vmx.HCPhysVirtApic);
858	if (RT_FAILURE(rc))
859	goto cleanup;
860	}
861
862	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
864	{
865	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
866	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867	if (RT_FAILURE(rc))
868	goto cleanup;
869	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
870	}
871
872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
873	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
874	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875	if (RT_FAILURE(rc))
876	goto cleanup;
877
878	/* Allocate the VM-exit MSR-load page for the host MSRs. */
879	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	#endif
883	}
884
885	return VINF_SUCCESS;
886
887	cleanup:
888	hmR0VmxStructsFree(pVM);
889	return rc;
890	}
891
892
893	/**
894	* Does global VT-x initialization (called during module initialization).
895	*
896	* @returns VBox status code.
897	*/
898	VMMR0DECL(int) VMXR0GlobalInit(void)
899	{
900	#ifdef HMVMX_USE_FUNCTION_TABLE
901	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
902	# ifdef VBOX_STRICT
903	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
904	Assert(g_apfnVMExitHandlers[i]);
905	# endif
906	#endif
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Does global VT-x termination (called during module termination).
913	*/
914	VMMR0DECL(void) VMXR0GlobalTerm()
915	{
916	/* Nothing to do currently. */
917	}
918
919
920	/**
921	* Sets up and activates VT-x on the current CPU.
922	*
923	* @returns VBox status code.
924	* @param pCpu Pointer to the global CPU info struct.
925	* @param pVM Pointer to the VM (can be NULL after a host resume
926	* operation).
927	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
928	* fEnabledByHost is true).
929	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
930	* @a fEnabledByHost is true).
931	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
932	* enable VT-x on the host.
933	*/
934	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
935	{
936	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
937	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
938
939	if (!fEnabledByHost)
940	{
941	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
942	if (RT_FAILURE(rc))
943	return rc;
944	}
945
946	/*
947	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
948	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
949	* each time while reusing a VPID after hitting the MaxASID limit once.
950	*/
951	if ( pVM
952	&& pVM->hm.s.fNestedPaging)
953	{
954	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
955	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
956	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
957	pCpu->fFlushAsidBeforeUse = false;
958	}
959	else
960	{
961	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
962	* without Nested Paging triggered this function) we still have the risk
963	* of potentially running with stale TLB-entries from other hypervisors
964	* when later we use a VM with NestedPaging. To fix this properly we will
965	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
966	* 'vmx_ept_vpid_caps' from it. Sigh. */
967	pCpu->fFlushAsidBeforeUse = true;
968	}
969
970	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
971	++pCpu->cTlbFlushes;
972
973	return VINF_SUCCESS;
974	}
975
976
977	/**
978	* Deactivates VT-x on the current CPU.
979	*
980	* @returns VBox status code.
981	* @param pCpu Pointer to the global CPU info struct.
982	* @param pvCpuPage Pointer to the VMXON region.
983	* @param HCPhysCpuPage Physical address of the VMXON region.
984	*
985	* @remarks This function should never be called when SUPR0EnableVTx() or
986	* similar was used to enable VT-x on the host.
987	*/
988	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
989	{
990	NOREF(pCpu);
991	NOREF(pvCpuPage);
992	NOREF(HCPhysCpuPage);
993
994	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
995	return hmR0VmxLeaveRootMode();
996	}
997
998
999	/**
1000	* Sets the permission bits for the specified MSR in the MSR bitmap.
1001	*
1002	* @param pVCpu Pointer to the VMCPU.
1003	* @param uMSR The MSR value.
1004	* @param enmRead Whether reading this MSR causes a VM-exit.
1005	* @param enmWrite Whether writing this MSR causes a VM-exit.
1006	*/
1007	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1008	{
1009	int32_t iBit;
1010	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1011
1012	/*
1013	* Layout:
1014	* 0x000 - 0x3ff - Low MSR read bits
1015	* 0x400 - 0x7ff - High MSR read bits
1016	* 0x800 - 0xbff - Low MSR write bits
1017	* 0xc00 - 0xfff - High MSR write bits
1018	*/
1019	if (uMsr <= 0x00001FFF)
1020	iBit = uMsr;
1021	else if ( uMsr >= 0xC0000000
1022	&& uMsr <= 0xC0001FFF)
1023	{
1024	iBit = (uMsr - 0xC0000000);
1025	pbMsrBitmap += 0x400;
1026	}
1027	else
1028	{
1029	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1030	return;
1031	}
1032
1033	Assert(iBit <= 0x1fff);
1034	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1035	ASMBitSet(pbMsrBitmap, iBit);
1036	else
1037	ASMBitClear(pbMsrBitmap, iBit);
1038
1039	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1040	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1041	else
1042	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1043	}
1044
1045
1046	/**
1047	* Flushes the TLB using EPT.
1048	*
1049	* @returns VBox status code.
1050	* @param pVM Pointer to the VM.
1051	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1052	* enmFlush).
1053	* @param enmFlush Type of flush.
1054	*/
1055	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1056	{
1057	AssertPtr(pVM);
1058	Assert(pVM->hm.s.fNestedPaging);
1059
1060	uint64_t descriptor[2];
1061	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1062	descriptor[0] = 0;
1063	else
1064	{
1065	Assert(pVCpu);
1066	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1067	}
1068	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1069
1070	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1071	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1072	rc));
1073	if ( RT_SUCCESS(rc)
1074	&& pVCpu)
1075	{
1076	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1077	}
1078	}
1079
1080
1081	/**
1082	* Flushes the TLB using VPID.
1083	*
1084	* @returns VBox status code.
1085	* @param pVM Pointer to the VM.
1086	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1087	* enmFlush).
1088	* @param enmFlush Type of flush.
1089	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1090	* on @a enmFlush).
1091	*/
1092	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1093	{
1094	AssertPtr(pVM);
1095	Assert(pVM->hm.s.vmx.fVpid);
1096
1097	uint64_t descriptor[2];
1098	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1099	{
1100	descriptor[0] = 0;
1101	descriptor[1] = 0;
1102	}
1103	else
1104	{
1105	AssertPtr(pVCpu);
1106	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1107	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1108	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1109	descriptor[1] = GCPtr;
1110	}
1111
1112	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1113	AssertMsg(rc == VINF_SUCCESS,
1114	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1115	if ( RT_SUCCESS(rc)
1116	&& pVCpu)
1117	{
1118	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1119	}
1120	}
1121
1122
1123	/**
1124	* Invalidates a guest page by guest virtual address. Only relevant for
1125	* EPT/VPID, otherwise there is nothing really to invalidate.
1126	*
1127	* @returns VBox status code.
1128	* @param pVM Pointer to the VM.
1129	* @param pVCpu Pointer to the VMCPU.
1130	* @param GCVirt Guest virtual address of the page to invalidate.
1131	*/
1132	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1133	{
1134	AssertPtr(pVM);
1135	AssertPtr(pVCpu);
1136	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1137
1138	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1139	if (!fFlushPending)
1140	{
1141	/*
1142	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1143	* See @bugref{6043} and @bugref{6177}.
1144	*
1145	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1146	* function maybe called in a loop with individual addresses.
1147	*/
1148	if (pVM->hm.s.vmx.fVpid)
1149	{
1150	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1151	{
1152	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1153	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1154	}
1155	else
1156	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1157	}
1158	else if (pVM->hm.s.fNestedPaging)
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	}
1161
1162	return VINF_SUCCESS;
1163	}
1164
1165
1166	/**
1167	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1168	* otherwise there is nothing really to invalidate.
1169	*
1170	* @returns VBox status code.
1171	* @param pVM Pointer to the VM.
1172	* @param pVCpu Pointer to the VMCPU.
1173	* @param GCPhys Guest physical address of the page to invalidate.
1174	*/
1175	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1176	{
1177	LogFlowFunc(("%RGp\n", GCPhys));
1178
1179	/*
1180	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1181	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1182	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1183	*/
1184	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1186	return VINF_SUCCESS;
1187	}
1188
1189
1190	/**
1191	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1192	* case where neither EPT nor VPID is supported by the CPU.
1193	*
1194	* @param pVM Pointer to the VM.
1195	* @param pVCpu Pointer to the VMCPU.
1196	*
1197	* @remarks Called with interrupts disabled.
1198	*/
1199	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1200	{
1201	NOREF(pVM);
1202	AssertPtr(pVCpu);
1203	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1204	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1205
1206	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1207	AssertPtr(pCpu);
1208
1209	pVCpu->hm.s.TlbShootdown.cPages = 0;
1210	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1211	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1212	pVCpu->hm.s.fForceTLBFlush = false;
1213	return;
1214	}
1215
1216
1217	/**
1218	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1219	*
1220	* @param pVM Pointer to the VM.
1221	* @param pVCpu Pointer to the VMCPU.
1222	* @remarks All references to "ASID" in this function pertains to "VPID" in
1223	* Intel's nomenclature. The reason is, to avoid confusion in compare
1224	* statements since the host-CPU copies are named "ASID".
1225	*
1226	* @remarks Called with interrupts disabled.
1227	*/
1228	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1229	{
1230	AssertPtr(pVM);
1231	AssertPtr(pVCpu);
1232	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1233	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1234	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1235
1236	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1237	AssertPtr(pCpu);
1238
1239	/*
1240	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1241	* This can happen both for start & resume due to long jumps back to ring-3.
1242	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1243	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1244	*/
1245	bool fNewASID = false;
1246	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1247	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1248	{
1249	pVCpu->hm.s.fForceTLBFlush = true;
1250	fNewASID = true;
1251	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1252	}
1253
1254	/*
1255	* Check for explicit TLB shootdowns.
1256	*/
1257	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1258	{
1259	pVCpu->hm.s.fForceTLBFlush = true;
1260	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1261	}
1262
1263	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1264	if (pVCpu->hm.s.fForceTLBFlush)
1265	{
1266	if (fNewASID)
1267	{
1268	++pCpu->uCurrentAsid;
1269	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1270	{
1271	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1272	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1273	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1274	}
1275
1276	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1277	if (pCpu->fFlushAsidBeforeUse)
1278	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1279	}
1280	else
1281	{
1282	/*
1283	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1284	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1285	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1286	* but not guest-physical mappings.
1287	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1288	*/
1289	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1290	}
1291
1292	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1293	pVCpu->hm.s.fForceTLBFlush = false;
1294	}
1295	else
1296	{
1297	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1298	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1299	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1300	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1301
1302	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1303	* not be executed. See hmQueueInvlPage() where it is commented
1304	* out. Support individual entry flushing someday. */
1305	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1306	{
1307	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1308
1309	/*
1310	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1311	* as supported by the CPU.
1312	*/
1313	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1314	{
1315	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1316	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1317	}
1318	else
1319	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1320	}
1321	else
1322	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1323	}
1324	pVCpu->hm.s.TlbShootdown.cPages = 0;
1325	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1326
1327	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1328	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1329	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1330	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1331	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1332	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1333
1334	/* Update VMCS with the VPID. */
1335	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1336	AssertRC(rc);
1337	}
1338
1339
1340	/**
1341	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1342	*
1343	* @returns VBox status code.
1344	* @param pVM Pointer to the VM.
1345	* @param pVCpu Pointer to the VMCPU.
1346	*
1347	* @remarks Called with interrupts disabled.
1348	*/
1349	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1350	{
1351	AssertPtr(pVM);
1352	AssertPtr(pVCpu);
1353	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1354	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1355
1356	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1357	AssertPtr(pCpu);
1358
1359	/*
1360	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1361	* This can happen both for start & resume due to long jumps back to ring-3.
1362	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1363	*/
1364	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1365	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1366	{
1367	pVCpu->hm.s.fForceTLBFlush = true;
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1369	}
1370
1371	/* Check for explicit TLB shootdown flushes. */
1372	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1373	{
1374	pVCpu->hm.s.fForceTLBFlush = true;
1375	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1376	}
1377
1378	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1379	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1380
1381	if (pVCpu->hm.s.fForceTLBFlush)
1382	{
1383	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1384	pVCpu->hm.s.fForceTLBFlush = false;
1385	}
1386	else
1387	{
1388	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1389	* not be executed. See hmQueueInvlPage() where it is commented
1390	* out. Support individual entry flushing someday. */
1391	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1392	{
1393	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1394	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1395	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1396	}
1397	else
1398	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1399	}
1400
1401	pVCpu->hm.s.TlbShootdown.cPages = 0;
1402	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1403	}
1404
1405
1406	/**
1407	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1408	*
1409	* @returns VBox status code.
1410	* @param pVM Pointer to the VM.
1411	* @param pVCpu Pointer to the VMCPU.
1412	*
1413	* @remarks Called with interrupts disabled.
1414	*/
1415	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1416	{
1417	AssertPtr(pVM);
1418	AssertPtr(pVCpu);
1419	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1420	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1421
1422	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1423
1424	/*
1425	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1426	* This can happen both for start & resume due to long jumps back to ring-3.
1427	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1428	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1429	*/
1430	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1431	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1432	{
1433	pVCpu->hm.s.fForceTLBFlush = true;
1434	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1435	}
1436
1437	/* Check for explicit TLB shootdown flushes. */
1438	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1439	{
1440	/*
1441	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1442	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1443	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1444	*/
1445	pVCpu->hm.s.fForceTLBFlush = true;
1446	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447	}
1448
1449	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450	if (pVCpu->hm.s.fForceTLBFlush)
1451	{
1452	++pCpu->uCurrentAsid;
1453	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1454	{
1455	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1456	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1457	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1458	}
1459
1460	pVCpu->hm.s.fForceTLBFlush = false;
1461	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1462	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1463	if (pCpu->fFlushAsidBeforeUse)
1464	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1465	}
1466	else
1467	{
1468	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1469	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1470	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1471	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1472
1473	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1474	* not be executed. See hmQueueInvlPage() where it is commented
1475	* out. Support individual entry flushing someday. */
1476	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1477	{
1478	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1479	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1480	{
1481	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1482	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1483	}
1484	else
1485	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1486	}
1487	else
1488	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1489	}
1490
1491	pVCpu->hm.s.TlbShootdown.cPages = 0;
1492	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1493
1494	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1495	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1496	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1497	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1498	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1499	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1500
1501	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1502	AssertRC(rc);
1503	}
1504
1505
1506	/**
1507	* Flushes the guest TLB entry based on CPU capabilities.
1508	*
1509	* @param pVCpu Pointer to the VMCPU.
1510	*/
1511	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1512	{
1513	PVM pVM = pVCpu->CTX_SUFF(pVM);
1514	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1515	{
1516	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1517	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1518	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1519	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1520	default:
1521	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1522	break;
1523	}
1524	}
1525
1526
1527	/**
1528	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1529	* TLB entries from the host TLB before VM-entry.
1530	*
1531	* @returns VBox status code.
1532	* @param pVM Pointer to the VM.
1533	*/
1534	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1535	{
1536	/*
1537	* Determine optimal flush type for Nested Paging.
1538	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1539	* guest execution (see hmR3InitFinalizeR0()).
1540	*/
1541	if (pVM->hm.s.fNestedPaging)
1542	{
1543	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1544	{
1545	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1546	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1547	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1548	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1549	else
1550	{
1551	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1552	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1553	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1554	}
1555
1556	/* Make sure the write-back cacheable memory type for EPT is supported. */
1557	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1558	{
1559	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1560	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1562	}
1563	}
1564	else
1565	{
1566	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1567	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1568	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1569	}
1570	}
1571
1572	/*
1573	* Determine optimal flush type for VPID.
1574	*/
1575	if (pVM->hm.s.vmx.fVpid)
1576	{
1577	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1578	{
1579	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1580	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1581	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1582	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1583	else
1584	{
1585	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1586	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1587	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1588	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1589	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1590	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1591	pVM->hm.s.vmx.fVpid = false;
1592	}
1593	}
1594	else
1595	{
1596	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1597	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1598	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1599	pVM->hm.s.vmx.fVpid = false;
1600	}
1601	}
1602
1603	/*
1604	* Setup the handler for flushing tagged-TLBs.
1605	*/
1606	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1607	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1608	else if (pVM->hm.s.fNestedPaging)
1609	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1610	else if (pVM->hm.s.vmx.fVpid)
1611	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1612	else
1613	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1614	return VINF_SUCCESS;
1615	}
1616
1617
1618	/**
1619	* Sets up pin-based VM-execution controls in the VMCS.
1620	*
1621	* @returns VBox status code.
1622	* @param pVM Pointer to the VM.
1623	* @param pVCpu Pointer to the VMCPU.
1624	*/
1625	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1626	{
1627	AssertPtr(pVM);
1628	AssertPtr(pVCpu);
1629
1630	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1631	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1632
1633	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1634	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1635	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1636
1637	/* Enable the VMX preemption timer. */
1638	if (pVM->hm.s.vmx.fUsePreemptTimer)
1639	{
1640	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1641	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1642	}
1643
1644	if ((val & zap) != val)
1645	{
1646	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1647	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1648	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1649	}
1650
1651	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1652	AssertRCReturn(rc, rc);
1653
1654	/* Update VCPU with the currently set pin-based VM-execution controls. */
1655	pVCpu->hm.s.vmx.u32PinCtls = val;
1656	return rc;
1657	}
1658
1659
1660	/**
1661	* Sets up processor-based VM-execution controls in the VMCS.
1662	*
1663	* @returns VBox status code.
1664	* @param pVM Pointer to the VM.
1665	* @param pVMCPU Pointer to the VMCPU.
1666	*/
1667	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1668	{
1669	AssertPtr(pVM);
1670	AssertPtr(pVCpu);
1671
1672	int rc = VERR_INTERNAL_ERROR_5;
1673	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1674	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1675
1676	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1677	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1678	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1679	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1681	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1682	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1683
1684	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1685	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1686	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1687	{
1688	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1690	}
1691
1692	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1693	if (!pVM->hm.s.fNestedPaging)
1694	{
1695	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1696	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1697	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1698	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1699	}
1700
1701	/* Use TPR shadowing if supported by the CPU. */
1702	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1703	{
1704	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1705	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1706	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1707	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1708	AssertRCReturn(rc, rc);
1709
1710	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1711	/* CR8 writes causes a VM-exit based on TPR threshold. */
1712	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1713	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1714	}
1715	else
1716	{
1717	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1718	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1719	}
1720
1721	/* Use MSR-bitmaps if supported by the CPU. */
1722	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1723	{
1724	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1725
1726	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1727	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1728	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1729	AssertRCReturn(rc, rc);
1730
1731	/*
1732	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1733	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1734	*/
1735	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1736	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1737	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1738	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1739	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1740	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1743	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1744	}
1745
1746	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1747	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1748	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1749
1750	if ((val & zap) != val)
1751	{
1752	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1753	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1754	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1755	}
1756
1757	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1758	AssertRCReturn(rc, rc);
1759
1760	/* Update VCPU with the currently set processor-based VM-execution controls. */
1761	pVCpu->hm.s.vmx.u32ProcCtls = val;
1762
1763	/*
1764	* Secondary processor-based VM-execution controls.
1765	*/
1766	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1767	{
1768	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1769	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1770
1771	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1772	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1773
1774	if (pVM->hm.s.fNestedPaging)
1775	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1776	else
1777	{
1778	/*
1779	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1780	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1781	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1782	*/
1783	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1784	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1785	}
1786
1787	if (pVM->hm.s.vmx.fVpid)
1788	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1789
1790	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1791	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1792
1793	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1794	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1795	* done dynamically. */
1796	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1797	{
1798	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1799	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1800	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1801	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1802	AssertRCReturn(rc, rc);
1803	}
1804
1805	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1806	{
1807	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1808	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1809	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810	}
1811
1812	if ((val & zap) != val)
1813	{
1814	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1815	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1816	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1817	}
1818
1819	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1820	AssertRCReturn(rc, rc);
1821
1822	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1823	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1824	}
1825
1826	return VINF_SUCCESS;
1827	}
1828
1829
1830	/**
1831	* Sets up miscellaneous (everything other than Pin & Processor-based
1832	* VM-execution) control fields in the VMCS.
1833	*
1834	* @returns VBox status code.
1835	* @param pVM Pointer to the VM.
1836	* @param pVCpu Pointer to the VMCPU.
1837	*/
1838	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1839	{
1840	AssertPtr(pVM);
1841	AssertPtr(pVCpu);
1842
1843	/** @todo Shouldn't we able to avoid initializing with 0? */
1844	int rc = VERR_GENERAL_FAILURE;
1845
1846	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1847	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1848	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1849
1850	/*
1851	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1852	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1853	* We thus use the exception bitmap to control it rather than use both.
1854	*/
1855	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1856	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1857
1858	/** @todo Explore possibility of using IO-bitmaps. */
1859	/* All IO & IOIO instructions cause VM-exits. */
1860	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1861	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1862
1863	/* Initialize the MSR-bitmap area. */
1864	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1865	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1866	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1867
1868	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1869	/* Setup MSR autoloading/storing. */
1870	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1871	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1872	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1873	AssertRCReturn(rc, rc);
1874	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1875	AssertRCReturn(rc, rc);
1876
1877	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1878	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1879	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1880	AssertRCReturn(rc, rc);
1881	#endif
1882
1883	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1884	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1885	AssertRCReturn(rc, rc);
1886
1887	/* Setup debug controls */
1888	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1889	AssertRCReturn(rc, rc);
1890	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1891	AssertRCReturn(rc, rc);
1892	return rc;
1893	}
1894
1895
1896	/**
1897	* Sets up the initial exception bitmap in the VMCS based on static conditions
1898	* (i.e. conditions that cannot ever change at runtime).
1899	*
1900	* @returns VBox status code.
1901	* @param pVM Pointer to the VM.
1902	* @param pVCpu Pointer to the VMCPU.
1903	*/
1904	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1905	{
1906	AssertPtr(pVM);
1907	AssertPtr(pVCpu);
1908
1909	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1910
1911	uint32_t u32XcptBitmap = 0;
1912
1913	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1914	if (!pVM->hm.s.fNestedPaging)
1915	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1916
1917	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1918	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1919	AssertRCReturn(rc, rc);
1920	return rc;
1921	}
1922
1923
1924	/**
1925	* Sets up the initial guest-state mask. The guest-state mask is consulted
1926	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1927	* for the nested virtualization case (as it would cause a VM-exit).
1928	*
1929	* @param pVCpu Pointer to the VMCPU.
1930	*/
1931	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1932	{
1933	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1934	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1935	return VINF_SUCCESS;
1936	}
1937
1938
1939	/**
1940	* Does per-VM VT-x initialization.
1941	*
1942	* @returns VBox status code.
1943	* @param pVM Pointer to the VM.
1944	*/
1945	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1946	{
1947	LogFlowFunc(("pVM=%p\n", pVM));
1948
1949	int rc = hmR0VmxStructsAlloc(pVM);
1950	if (RT_FAILURE(rc))
1951	{
1952	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1953	return rc;
1954	}
1955
1956	return VINF_SUCCESS;
1957	}
1958
1959
1960	/**
1961	* Does per-VM VT-x termination.
1962	*
1963	* @returns VBox status code.
1964	* @param pVM Pointer to the VM.
1965	*/
1966	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1967	{
1968	LogFlowFunc(("pVM=%p\n", pVM));
1969
1970	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1971	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1972	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1973	#endif
1974	hmR0VmxStructsFree(pVM);
1975	return VINF_SUCCESS;
1976	}
1977
1978
1979	/**
1980	* Sets up the VM for execution under VT-x.
1981	* This function is only called once per-VM during initalization.
1982	*
1983	* @returns VBox status code.
1984	* @param pVM Pointer to the VM.
1985	*/
1986	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1987	{
1988	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1989	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1990
1991	LogFlowFunc(("pVM=%p\n", pVM));
1992
1993	/*
1994	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1995	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1996	*/
1997	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
1998	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1999	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2000	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2001	{
2002	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2003	return VERR_INTERNAL_ERROR;
2004	}
2005
2006	/* Initialize these always, see hmR3InitFinalizeR0().*/
2007	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2008	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2009
2010	/* Setup the tagged-TLB flush handlers. */
2011	int rc = hmR0VmxSetupTaggedTlb(pVM);
2012	if (RT_FAILURE(rc))
2013	{
2014	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2015	return rc;
2016	}
2017
2018	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2019	{
2020	PVMCPU pVCpu = &pVM->aCpus[i];
2021	AssertPtr(pVCpu);
2022	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2023
2024	/* Set revision dword at the beginning of the VMCS structure. */
2025	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2026
2027	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2028	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2029	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2030	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2031
2032	/* Load this VMCS as the current VMCS. */
2033	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2034	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2035	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2036
2037	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2038	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2040
2041	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2042	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2043	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2044
2045	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2046	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2047	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2048
2049	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2050	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2051	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2052
2053	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2054	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2055	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2056
2057	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2058	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2059	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2060	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2061	#endif
2062
2063	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2064	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2065	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2066	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2067
2068	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2069	}
2070
2071	return VINF_SUCCESS;
2072	}
2073
2074
2075	/**
2076	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2077	* the VMCS.
2078	*
2079	* @returns VBox status code.
2080	* @param pVM Pointer to the VM.
2081	* @param pVCpu Pointer to the VMCPU.
2082	*/
2083	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2084	{
2085	RTCCUINTREG uReg = ASMGetCR0();
2086	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2087	AssertRCReturn(rc, rc);
2088
2089	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2090	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2091	if (HMVMX_IS_64BIT_HOST_MODE())
2092	{
2093	uint64_t uRegCR3 = HMR0Get64bitCR3();
2094	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2095	}
2096	else
2097	#endif
2098	{
2099	uReg = ASMGetCR3();
2100	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2101	}
2102	AssertRCReturn(rc, rc);
2103
2104	uReg = ASMGetCR4();
2105	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2106	AssertRCReturn(rc, rc);
2107	return rc;
2108	}
2109
2110
2111	/**
2112	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2113	* the host-state area in the VMCS.
2114	*
2115	* @returns VBox status code.
2116	* @param pVM Pointer to the VM.
2117	* @param pVCpu Pointer to the VMCPU.
2118	*/
2119	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2120	{
2121	int rc = VERR_INTERNAL_ERROR_5;
2122	RTSEL uSelDS = 0;
2123	RTSEL uSelES = 0;
2124	RTSEL uSelFS = 0;
2125	RTSEL uSelGS = 0;
2126	RTSEL uSelTR = 0;
2127
2128	/*
2129	* Host DS, ES, FS and GS segment registers.
2130	*/
2131	#if HC_ARCH_BITS == 64
2132	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2133	uSelDS = ASMGetDS();
2134	uSelES = ASMGetES();
2135	uSelFS = ASMGetFS();
2136	uSelGS = ASMGetGS();
2137	#endif
2138
2139	/*
2140	* Host CS and SS segment registers.
2141	*/
2142	RTSEL uSelCS;
2143	RTSEL uSelSS;
2144	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2145	if (HMVMX_IS_64BIT_HOST_MODE())
2146	{
2147	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2148	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2149	}
2150	else
2151	{
2152	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2153	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2154	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2155	}
2156	#else
2157	uSelCS = ASMGetCS();
2158	uSelSS = ASMGetSS();
2159	#endif
2160
2161	/*
2162	* Host TR segment register.
2163	*/
2164	uSelTR = ASMGetTR();
2165
2166	#if HC_ARCH_BITS == 64
2167	/*
2168	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2169	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2170	*/
2171	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2172	{
2173	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2174	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2175	uSelDS = 0;
2176	}
2177	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2178	{
2179	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2180	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2181	uSelES = 0;
2182	}
2183	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2184	{
2185	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2186	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2187	uSelFS = 0;
2188	}
2189	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2190	{
2191	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2192	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2193	uSelGS = 0;
2194	}
2195	#endif
2196
2197	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2198	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2199	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2200	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2201	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2202	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2203	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2204	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2205	Assert(uSelCS);
2206	Assert(uSelTR);
2207
2208	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2209	#if 0
2210	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2211	Assert(uSelSS != 0);
2212	#endif
2213
2214	/* Write these host selector fields into the host-state area in the VMCS. */
2215	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2216	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2217	#if HC_ARCH_BITS == 64
2218	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2219	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2220	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2221	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2222	#endif
2223	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2224
2225	/*
2226	* Host GDTR and IDTR.
2227	*/
2228	RTGDTR Gdtr;
2229	RT_ZERO(Gdtr);
2230	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2231	if (HMVMX_IS_64BIT_HOST_MODE())
2232	{
2233	X86XDTR64 Gdtr64;
2234	X86XDTR64 Idtr64;
2235	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2236	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2237	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2238
2239	Gdtr.cbGdt = Gdtr64.cb;
2240	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2241	}
2242	else
2243	#endif
2244	{
2245	RTIDTR Idtr;
2246	ASMGetGDTR(&Gdtr);
2247	ASMGetIDTR(&Idtr);
2248	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2249	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2250
2251	#if HC_ARCH_BITS == 64
2252	/*
2253	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2254	* maximum limit (0xffff) on every VM-exit.
2255	*/
2256	if (Gdtr.cbGdt != 0xffff)
2257	{
2258	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2259	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2260	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2261	}
2262
2263	/*
2264	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2265	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2266	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2267	*/
2268	if (Idtr.cbIdt < 0x0fff)
2269	{
2270	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2271	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2272	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2273	}
2274	#endif
2275	}
2276
2277	/*
2278	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2279	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2280	*/
2281	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2282	{
2283	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2284	return VERR_VMX_INVALID_HOST_STATE;
2285	}
2286
2287	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2288	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2289	if (HMVMX_IS_64BIT_HOST_MODE())
2290	{
2291	/* We need the 64-bit TR base for hybrid darwin. */
2292	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2293	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2294	}
2295	else
2296	#endif
2297	{
2298	uintptr_t uTRBase;
2299	#if HC_ARCH_BITS == 64
2300	uTRBase = X86DESC64_BASE(pDesc);
2301	#else
2302	uTRBase = X86DESC_BASE(pDesc);
2303	#endif
2304	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2305	}
2306	AssertRCReturn(rc, rc);
2307
2308	/*
2309	* Host FS base and GS base.
2310	*/
2311	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2312	if (HMVMX_IS_64BIT_HOST_MODE())
2313	{
2314	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2315	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2316	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2317	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2318
2319	# if HC_ARCH_BITS == 64
2320	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2321	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2322	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2323	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2324	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2325	# endif
2326	}
2327	#endif
2328	return rc;
2329	}
2330
2331
2332	/**
2333	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2334	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2335	* the host after every successful VM exit.
2336	*
2337	* @returns VBox status code.
2338	* @param pVM Pointer to the VM.
2339	* @param pVCpu Pointer to the VMCPU.
2340	*/
2341	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2342	{
2343	AssertPtr(pVCpu);
2344	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2345
2346	int rc = VINF_SUCCESS;
2347	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2348	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2349	uint32_t cHostMsrs = 0;
2350	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2351
2352	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2353	{
2354	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2355
2356	# if HC_ARCH_BITS == 64
2357	/* Paranoia. 64-bit code requires these bits to be set always. */
2358	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2359
2360	/*
2361	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2362	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2363	* some reason (e.g. allow transparent reads) we would activate the code below.
2364	*/
2365	# if 0
2366	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2367	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2368	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2369	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2370	if (CPUMIsGuestInLongMode(pVCpu))
2371	{
2372	uint64_t u64GuestEfer;
2373	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2374	AssertRC(rc);
2375
2376	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2377	{
2378	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2379	pHostMsr->u32Reserved = 0;
2380	pHostMsr->u64Value = u64HostEfer;
2381	pHostMsr++; cHostMsrs++;
2382	}
2383	}
2384	# endif
2385	# else /* HC_ARCH_BITS != 64 */
2386	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2387	pHostMsr->u32Reserved = 0;
2388	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2389	if (CPUMIsGuestInLongMode(pVCpu))
2390	{
2391	/* Must match the EFER value in our 64 bits switcher. */
2392	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2393	}
2394	else
2395	# endif
2396	pHostMsr->u64Value = u64HostEfer;
2397	pHostMsr++; cHostMsrs++;
2398	# endif /* HC_ARCH_BITS == 64 */
2399	}
2400
2401	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2402	if (HMVMX_IS_64BIT_HOST_MODE())
2403	{
2404	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2405	pHostMsr->u32Reserved = 0;
2406	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2407	pHostMsr++; cHostMsrs++;
2408	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2409	pHostMsr->u32Reserved = 0;
2410	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2411	pHostMsr++; cHostMsrs++;
2412	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2413	pHostMsr->u32Reserved = 0;
2414	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2415	pHostMsr++; cHostMsrs++;
2416	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2417	pHostMsr->u32Reserved = 0;
2418	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2419	pHostMsr++; cHostMsrs++;
2420	}
2421	# endif
2422
2423	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2424	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2425	{
2426	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2431	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2432
2433	/*
2434	* Host Sysenter MSRs.
2435	*/
2436	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2437	AssertRCReturn(rc, rc);
2438	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2439	if (HMVMX_IS_64BIT_HOST_MODE())
2440	{
2441	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2442	AssertRCReturn(rc, rc);
2443	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2444	}
2445	else
2446	{
2447	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2448	AssertRCReturn(rc, rc);
2449	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2450	}
2451	#elif HC_ARCH_BITS == 32
2452	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2453	AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2455	#else
2456	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2457	AssertRCReturn(rc, rc);
2458	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2459	#endif
2460	AssertRCReturn(rc, rc);
2461
2462	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2463	* hmR0VmxSetupExitCtls() !! */
2464	return rc;
2465	}
2466
2467
2468	/**
2469	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2470	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2471	* controls".
2472	*
2473	* @returns VBox status code.
2474	* @param pVCpu Pointer to the VMCPU.
2475	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2476	* out-of-sync. Make sure to update the required fields
2477	* before using them.
2478	*
2479	* @remarks No-long-jump zone!!!
2480	*/
2481	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2482	{
2483	int rc = VINF_SUCCESS;
2484	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2485	{
2486	PVM pVM = pVCpu->CTX_SUFF(pVM);
2487	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2488	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2489
2490	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2491	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2492
2493	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2494	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2495	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2496	else
2497	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2498
2499	/*
2500	* The following should not be set (since we're not in SMM mode):
2501	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2502	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2503	*/
2504
2505	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2506	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2507	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2508
2509	if ((val & zap) != val)
2510	{
2511	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2512	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2513	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2514	}
2515
2516	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2517	AssertRCReturn(rc, rc);
2518
2519	/* Update VCPU with the currently set VM-exit controls. */
2520	pVCpu->hm.s.vmx.u32EntryCtls = val;
2521	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2522	}
2523	return rc;
2524	}
2525
2526
2527	/**
2528	* Sets up the VM-exit controls in the VMCS.
2529	*
2530	* @returns VBox status code.
2531	* @param pVM Pointer to the VM.
2532	* @param pVCpu Pointer to the VMCPU.
2533	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2534	* out-of-sync. Make sure to update the required fields
2535	* before using them.
2536	*
2537	* @remarks requires EFER.
2538	*/
2539	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2540	{
2541	int rc = VINF_SUCCESS;
2542	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2543	{
2544	PVM pVM = pVCpu->CTX_SUFF(pVM);
2545	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2546	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2547
2548	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2549	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2550
2551	/*
2552	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2553	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2554	*/
2555	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2556	if (HMVMX_IS_64BIT_HOST_MODE())
2557	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2558	else
2559	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2560	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2561	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2562	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2563	else
2564	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2565	#endif
2566
2567	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2568	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2569
2570	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2571	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2572	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2573	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2574	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2575
2576	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2577	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2578
2579	if ((val & zap) != val)
2580	{
2581	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2582	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2583	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2584	}
2585
2586	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2587	AssertRCReturn(rc, rc);
2588
2589	/* Update VCPU with the currently set VM-exit controls. */
2590	pVCpu->hm.s.vmx.u32ExitCtls = val;
2591	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2592	}
2593	return rc;
2594	}
2595
2596
2597	/**
2598	* Loads the guest APIC and related state.
2599	*
2600	* @returns VBox status code.
2601	* @param pVM Pointer to the VM.
2602	* @param pVCpu Pointer to the VMCPU.
2603	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2604	* out-of-sync. Make sure to update the required fields
2605	* before using them.
2606	*/
2607	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2608	{
2609	int rc = VINF_SUCCESS;
2610	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2611	{
2612	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2613	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2614	{
2615	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2616
2617	bool fPendingIntr = false;
2618	uint8_t u8Tpr = 0;
2619	uint8_t u8PendingIntr = 0;
2620	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2621	AssertRCReturn(rc, rc);
2622
2623	/*
2624	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2625	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2626	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2627	* the interrupt when we VM-exit for other reasons.
2628	*/
2629	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2630	uint32_t u32TprThreshold = 0;
2631	if (fPendingIntr)
2632	{
2633	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2634	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2635	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2636	if (u8PendingPriority <= u8TprPriority)
2637	u32TprThreshold = u8PendingPriority;
2638	else
2639	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2640	}
2641	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2642
2643	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2644	AssertRCReturn(rc, rc);
2645
2646	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2647	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2648	{
2649	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2650	pMixedCtx->msrLSTAR = u8Tpr;
2651	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2652	{
2653	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2654	if (fPendingIntr)
2655	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2656	else
2657	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2658	}
2659	}
2660	}
2661
2662	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2663	}
2664	return rc;
2665	}
2666
2667
2668	/**
2669	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2670	*
2671	* @returns
2672	* @param pVCpu Pointer to the VMCPU.
2673	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2674	* out-of-sync. Make sure to update the required fields
2675	* before using them.
2676	*
2677	* @remarks No-long-jump zone!!!
2678	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2679	*/
2680	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2681	{
2682	/*
2683	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2684	* inhibit interrupts or clear any existing interrupt-inhibition.
2685	*/
2686	uint32_t uIntrState = 0;
2687	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2688	{
2689	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2690	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2691	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2692	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2693	{
2694	/*
2695	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2696	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2697	*/
2698	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2699	}
2700	else if (pMixedCtx->eflags.Bits.u1IF)
2701	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2702	else
2703	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2704	}
2705	return uIntrState;
2706	}
2707
2708
2709	/**
2710	* Loads the guest's interruptibility-state into the guest-state area in the
2711	* VMCS.
2712	*
2713	* @returns VBox status code.
2714	* @param pVCpu Pointer to the VMCPU.
2715	* @param uIntrState The interruptibility-state to set.
2716	*/
2717	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2718	{
2719	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2720	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2721	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2722	AssertRCReturn(rc, rc);
2723	return rc;
2724	}
2725
2726
2727	/**
2728	* Loads the guest's RIP into the guest-state area in the VMCS.
2729	*
2730	* @returns VBox status code.
2731	* @param pVCpu Pointer to the VMCPU.
2732	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2733	* out-of-sync. Make sure to update the required fields
2734	* before using them.
2735	*
2736	* @remarks No-long-jump zone!!!
2737	*/
2738	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2739	{
2740	int rc = VINF_SUCCESS;
2741	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2742	{
2743	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2744	AssertRCReturn(rc, rc);
2745	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2746	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2747	}
2748	return rc;
2749	}
2750
2751
2752	/**
2753	* Loads the guest's RSP into the guest-state area in the VMCS.
2754	*
2755	* @returns VBox status code.
2756	* @param pVCpu Pointer to the VMCPU.
2757	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2758	* out-of-sync. Make sure to update the required fields
2759	* before using them.
2760	*
2761	* @remarks No-long-jump zone!!!
2762	*/
2763	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2764	{
2765	int rc = VINF_SUCCESS;
2766	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2767	{
2768	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2769	AssertRCReturn(rc, rc);
2770	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2771	}
2772	return rc;
2773	}
2774
2775
2776	/**
2777	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2778	*
2779	* @returns VBox status code.
2780	* @param pVCpu Pointer to the VMCPU.
2781	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2782	* out-of-sync. Make sure to update the required fields
2783	* before using them.
2784	*
2785	* @remarks No-long-jump zone!!!
2786	*/
2787	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2788	{
2789	int rc = VINF_SUCCESS;
2790	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2791	{
2792	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2793	Let us assert it as such and use 32-bit VMWRITE. */
2794	Assert(!(pMixedCtx->rflags.u64 >> 32));
2795	X86EFLAGS uEFlags = pMixedCtx->eflags;
2796	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2797	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2798
2799	/*
2800	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2801	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2802	*/
2803	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2804	{
2805	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2806	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2807	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2808	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2809	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2810	}
2811
2812	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2813	AssertRCReturn(rc, rc);
2814
2815	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2816	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2817	}
2818	return rc;
2819	}
2820
2821
2822	/**
2823	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2824	*
2825	* @returns VBox status code.
2826	* @param pVCpu Pointer to the VMCPU.
2827	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2828	* out-of-sync. Make sure to update the required fields
2829	* before using them.
2830	*
2831	* @remarks No-long-jump zone!!!
2832	*/
2833	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2834	{
2835	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2836	AssertRCReturn(rc, rc);
2837	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2838	AssertRCReturn(rc, rc);
2839	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2840	AssertRCReturn(rc, rc);
2841	return rc;
2842	}
2843
2844
2845	/**
2846	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2847	* in the VMCS.
2848	*
2849	* @returns VBox status code.
2850	* @param pVM Pointer to the VM.
2851	* @param pVCpu Pointer to the VMCPU.
2852	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2853	* out-of-sync. Make sure to update the required fields
2854	* before using them.
2855	*
2856	* @remarks No-long-jump zone!!!
2857	*/
2858	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2859	{
2860	int rc = VINF_SUCCESS;
2861	PVM pVM = pVCpu->CTX_SUFF(pVM);
2862
2863	/*
2864	* Guest CR0.
2865	* Guest FPU.
2866	*/
2867	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2868	{
2869	Assert(!(pCtx->cr0 >> 32));
2870	uint32_t u32GuestCR0 = pCtx->cr0;
2871
2872	/* The guest's view (read access) of its CR0 is unblemished. */
2873	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2874	AssertRCReturn(rc, rc);
2875	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2876
2877	/* Setup VT-x's view of the guest CR0. */
2878	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2879	if (pVM->hm.s.fNestedPaging)
2880	{
2881	if (CPUMIsGuestPagingEnabledEx(pCtx))
2882	{
2883	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2884	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2885	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2886	}
2887	else
2888	{
2889	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2890	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2891	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2892	}
2893
2894	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2895	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2896	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2897
2898	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2899	AssertRCReturn(rc, rc);
2900	}
2901	else
2902	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2903
2904	/*
2905	* Guest FPU bits.
2906	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2907	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2908	*/
2909	u32GuestCR0 \|= X86_CR0_NE;
2910	bool fInterceptNM = false;
2911	if (CPUMIsGuestFPUStateActive(pVCpu))
2912	{
2913	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2914	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2915	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2916	}
2917	else
2918	{
2919	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2920	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2921	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2922	}
2923
2924	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2925	bool fInterceptMF = false;
2926	if (!(pCtx->cr0 & X86_CR0_NE))
2927	fInterceptMF = true;
2928
2929	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2930	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2931	{
2932	Assert(PDMVmmDevHeapIsEnabled(pVM));
2933	Assert(pVM->hm.s.vmx.pRealModeTSS);
2934	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2935	fInterceptNM = true;
2936	fInterceptMF = true;
2937	}
2938	else
2939	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2940
2941	if (fInterceptNM)
2942	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2943	else
2944	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2945
2946	if (fInterceptMF)
2947	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2948	else
2949	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2950
2951	/* Additional intercepts for debugging, define these yourself explicitly. */
2952	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2953	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2954	\| RT_BIT(X86_XCPT_DB)
2955	\| RT_BIT(X86_XCPT_DE)
2956	\| RT_BIT(X86_XCPT_NM)
2957	\| RT_BIT(X86_XCPT_UD)
2958	\| RT_BIT(X86_XCPT_NP)
2959	\| RT_BIT(X86_XCPT_SS)
2960	\| RT_BIT(X86_XCPT_GP)
2961	\| RT_BIT(X86_XCPT_PF)
2962	\| RT_BIT(X86_XCPT_MF);
2963	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2964	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2965	#endif
2966
2967	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2968
2969	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2970	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2971	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2972	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2973	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2974	else
2975	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2976
2977	u32GuestCR0 \|= uSetCR0;
2978	u32GuestCR0 &= uZapCR0;
2979	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2980
2981	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2982	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2983	AssertRCReturn(rc, rc);
2984	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2985	AssertRCReturn(rc, rc);
2986	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2987
2988	/*
2989	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2990	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2991	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2992	*/
2993	uint32_t u32CR0Mask = 0;
2994	u32CR0Mask = X86_CR0_PE
2995	\| X86_CR0_NE
2996	\| X86_CR0_WP
2997	\| X86_CR0_PG
2998	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2999	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3000	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3001	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3002	u32CR0Mask &= ~X86_CR0_PE;
3003	if (pVM->hm.s.fNestedPaging)
3004	u32CR0Mask &= ~X86_CR0_WP;
3005
3006	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3007	if (fInterceptNM)
3008	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3009	else
3010	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3011
3012	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3013	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3014	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3015	AssertRCReturn(rc, rc);
3016
3017	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3018	}
3019
3020	/*
3021	* Guest CR2.
3022	* It's always loaded in the assembler code. Nothing to do here.
3023	*/
3024
3025	/*
3026	* Guest CR3.
3027	*/
3028	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3029	{
3030	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3031	if (pVM->hm.s.fNestedPaging)
3032	{
3033	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3034
3035	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3036	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3037	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3038	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3039
3040	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3041	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3042	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3043
3044	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3045	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3046	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3047	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3048
3049	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3050	AssertRCReturn(rc, rc);
3051	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3052
3053	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3054	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3055	{
3056	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3057	if (CPUMIsGuestInPAEModeEx(pCtx))
3058	{
3059	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3060	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3061	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3062	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3063	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3064	}
3065
3066	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3067	have Unrestricted Execution to handle the guest when it's not using paging. */
3068	GCPhysGuestCR3 = pCtx->cr3;
3069	}
3070	else
3071	{
3072	/*
3073	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3074	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3075	* EPT takes care of translating it to host-physical addresses.
3076	*/
3077	RTGCPHYS GCPhys;
3078	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3079	Assert(PDMVmmDevHeapIsEnabled(pVM));
3080
3081	/* We obtain it here every time as the guest could have relocated this PCI region. */
3082	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3083	AssertRCReturn(rc, rc);
3084
3085	GCPhysGuestCR3 = GCPhys;
3086	}
3087
3088	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3089	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3090	}
3091	else
3092	{
3093	/* Non-nested paging case, just use the hypervisor's CR3. */
3094	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3095
3096	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3097	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3098	}
3099	AssertRCReturn(rc, rc);
3100
3101	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3102	}
3103
3104	/*
3105	* Guest CR4.
3106	*/
3107	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3108	{
3109	Assert(!(pCtx->cr4 >> 32));
3110	uint32_t u32GuestCR4 = pCtx->cr4;
3111
3112	/* The guest's view of its CR4 is unblemished. */
3113	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3114	AssertRCReturn(rc, rc);
3115	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3116
3117	/* Setup VT-x's view of the guest CR4. */
3118	/*
3119	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3120	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3121	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3122	*/
3123	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3124	{
3125	Assert(pVM->hm.s.vmx.pRealModeTSS);
3126	Assert(PDMVmmDevHeapIsEnabled(pVM));
3127	u32GuestCR4 &= ~X86_CR4_VME;
3128	}
3129
3130	if (pVM->hm.s.fNestedPaging)
3131	{
3132	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3133	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3134	{
3135	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3136	u32GuestCR4 \|= X86_CR4_PSE;
3137	/* Our identity mapping is a 32 bits page directory. */
3138	u32GuestCR4 &= ~X86_CR4_PAE;
3139	}
3140	/* else use guest CR4.*/
3141	}
3142	else
3143	{
3144	/*
3145	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3146	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3147	*/
3148	switch (pVCpu->hm.s.enmShadowMode)
3149	{
3150	case PGMMODE_REAL: /* Real-mode. */
3151	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3152	case PGMMODE_32_BIT: /* 32-bit paging. */
3153	{
3154	u32GuestCR4 &= ~X86_CR4_PAE;
3155	break;
3156	}
3157
3158	case PGMMODE_PAE: /* PAE paging. */
3159	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3160	{
3161	u32GuestCR4 \|= X86_CR4_PAE;
3162	break;
3163	}
3164
3165	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3166	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3167	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3168	break;
3169	#endif
3170	default:
3171	AssertFailed();
3172	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3173	}
3174	}
3175
3176	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3177	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3178	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3179	u32GuestCR4 \|= uSetCR4;
3180	u32GuestCR4 &= uZapCR4;
3181
3182	/* Write VT-x's view of the guest CR4 into the VMCS. */
3183	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3184	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3185	AssertRCReturn(rc, rc);
3186
3187	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3188	uint32_t u32CR4Mask = 0;
3189	u32CR4Mask = X86_CR4_VME
3190	\| X86_CR4_PAE
3191	\| X86_CR4_PGE
3192	\| X86_CR4_PSE
3193	\| X86_CR4_VMXE;
3194	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3195	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3196	AssertRCReturn(rc, rc);
3197
3198	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3199	}
3200	return rc;
3201	}
3202
3203
3204	/**
3205	* Loads the guest debug registers into the guest-state area in the VMCS.
3206	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3207	*
3208	* @returns VBox status code.
3209	* @param pVCpu Pointer to the VMCPU.
3210	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3211	* out-of-sync. Make sure to update the required fields
3212	* before using them.
3213	*
3214	* @remarks No-long-jump zone!!!
3215	*/
3216	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3217	{
3218	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3219	return VINF_SUCCESS;
3220
3221	#ifdef VBOX_STRICT
3222	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3223	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3224	{
3225	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3226	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3227	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3228	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3229	}
3230	#endif
3231
3232	int rc = VERR_INTERNAL_ERROR_5;
3233	PVM pVM = pVCpu->CTX_SUFF(pVM);
3234	bool fInterceptDB = false;
3235	bool fInterceptMovDRx = false;
3236	if (DBGFIsStepping(pVCpu))
3237	{
3238	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3239	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3240	{
3241	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3242	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3243	AssertRCReturn(rc, rc);
3244	Assert(fInterceptDB == false);
3245	}
3246	else
3247	{
3248	fInterceptDB = true;
3249	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3250	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3251	}
3252	}
3253
3254	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3255	{
3256	if (!CPUMIsHyperDebugStateActive(pVCpu))
3257	{
3258	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3259	AssertRC(rc);
3260	}
3261	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3262	fInterceptMovDRx = true;
3263	}
3264	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3265	{
3266	if (!CPUMIsGuestDebugStateActive(pVCpu))
3267	{
3268	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3269	AssertRC(rc);
3270	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3271	}
3272	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3273	Assert(fInterceptMovDRx == false);
3274	}
3275	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3276	{
3277	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3278	fInterceptMovDRx = true;
3279	}
3280
3281	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3282	if (fInterceptDB)
3283	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3284	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3285	{
3286	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3287	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3288	#endif
3289	}
3290
3291	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3292	if (fInterceptMovDRx)
3293	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3294	else
3295	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3296
3297	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3298	AssertRCReturn(rc, rc);
3299	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3300	AssertRCReturn(rc, rc);
3301
3302	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3303	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3304	AssertRCReturn(rc, rc);
3305
3306	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3307	return rc;
3308	}
3309
3310
3311	#ifdef VBOX_STRICT
3312	/**
3313	* Strict function to validate segment registers.
3314	*
3315	* @remarks Requires CR0.
3316	*/
3317	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3318	{
3319	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3320	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3321	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3322
3323	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3324	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3325	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3326	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3327	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3328	{
3329	/* Protected mode checks */
3330	/* CS */
3331	Assert(pCtx->cs.Attr.n.u1Present);
3332	Assert(!(pCtx->cs.Attr.u & 0xf00));
3333	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3334	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3335	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3336	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3337	\|\| (pCtx->cs.Attr.n.u1Granularity));
3338	/* CS cannot be loaded with NULL in protected mode. */
3339	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3340	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3341	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3342	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3343	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3344	else
3345	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3346	/* SS */
3347	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3348	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3349	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3350	if ( !(pCtx->cr0 & X86_CR0_PE)
3351	\|\| pCtx->cs.Attr.n.u4Type == 3)
3352	{
3353	Assert(!pCtx->ss.Attr.n.u2Dpl);
3354	}
3355	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3356	{
3357	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3358	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3359	Assert(pCtx->ss.Attr.n.u1Present);
3360	Assert(!(pCtx->ss.Attr.u & 0xf00));
3361	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3362	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3363	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3364	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3365	\|\| (pCtx->ss.Attr.n.u1Granularity));
3366	}
3367	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3368	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3369	{
3370	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3371	Assert(pCtx->ds.Attr.n.u1Present);
3372	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3373	Assert(!(pCtx->ds.Attr.u & 0xf00));
3374	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3375	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3376	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3377	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3378	\|\| (pCtx->ds.Attr.n.u1Granularity));
3379	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3380	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3381	}
3382	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3383	{
3384	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3385	Assert(pCtx->es.Attr.n.u1Present);
3386	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3387	Assert(!(pCtx->es.Attr.u & 0xf00));
3388	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3389	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3390	\|\| !(pCtx->es.Attr.n.u1Granularity));
3391	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3392	\|\| (pCtx->es.Attr.n.u1Granularity));
3393	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3394	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3395	}
3396	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3397	{
3398	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3399	Assert(pCtx->fs.Attr.n.u1Present);
3400	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3401	Assert(!(pCtx->fs.Attr.u & 0xf00));
3402	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3403	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3404	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3405	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3406	\|\| (pCtx->fs.Attr.n.u1Granularity));
3407	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3408	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3409	}
3410	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3411	{
3412	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3413	Assert(pCtx->gs.Attr.n.u1Present);
3414	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3415	Assert(!(pCtx->gs.Attr.u & 0xf00));
3416	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3417	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3418	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3419	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3420	\|\| (pCtx->gs.Attr.n.u1Granularity));
3421	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3422	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3423	}
3424	/* 64-bit capable CPUs. */
3425	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3426	Assert(!(pCtx->cs.u64Base >> 32));
3427	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3428	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3429	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3430	# endif
3431	}
3432	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3433	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3434	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3435	{
3436	/* Real and v86 mode checks. */
3437	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3438	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3439	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3440	{
3441	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3442	}
3443	else
3444	{
3445	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3446	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3447	}
3448
3449	/* CS */
3450	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3451	Assert(pCtx->cs.u32Limit == 0xffff);
3452	Assert(u32CSAttr == 0xf3);
3453	/* SS */
3454	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3455	Assert(pCtx->ss.u32Limit == 0xffff);
3456	Assert(u32SSAttr == 0xf3);
3457	/* DS */
3458	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3459	Assert(pCtx->ds.u32Limit == 0xffff);
3460	Assert(u32DSAttr == 0xf3);
3461	/* ES */
3462	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3463	Assert(pCtx->es.u32Limit == 0xffff);
3464	Assert(u32ESAttr == 0xf3);
3465	/* FS */
3466	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3467	Assert(pCtx->fs.u32Limit == 0xffff);
3468	Assert(u32FSAttr == 0xf3);
3469	/* GS */
3470	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3471	Assert(pCtx->gs.u32Limit == 0xffff);
3472	Assert(u32GSAttr == 0xf3);
3473	/* 64-bit capable CPUs. */
3474	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3475	Assert(!(pCtx->cs.u64Base >> 32));
3476	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3477	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3478	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3479	# endif
3480	}
3481	}
3482	#endif /* VBOX_STRICT */
3483
3484
3485	/**
3486	* Writes a guest segment register into the guest-state area in the VMCS.
3487	*
3488	* @returns VBox status code.
3489	* @param pVCpu Pointer to the VMCPU.
3490	* @param idxSel Index of the selector in the VMCS.
3491	* @param idxLimit Index of the segment limit in the VMCS.
3492	* @param idxBase Index of the segment base in the VMCS.
3493	* @param idxAccess Index of the access rights of the segment in the VMCS.
3494	* @param pSelReg Pointer to the segment selector.
3495	* @param pCtx Pointer to the guest-CPU context.
3496	*
3497	* @remarks No-long-jump zone!!!
3498	*/
3499	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3500	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3501	{
3502	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3503	AssertRCReturn(rc, rc);
3504	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3505	AssertRCReturn(rc, rc);
3506	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3507	AssertRCReturn(rc, rc);
3508
3509	uint32_t u32Access = pSelReg->Attr.u;
3510	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3511	{
3512	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3513	u32Access = 0xf3;
3514	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3515	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3516	}
3517	else
3518	{
3519	/*
3520	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3521	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3522	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3523	* loaded in protected-mode have their attribute as 0.
3524	*/
3525	if (!u32Access)
3526	u32Access = HMVMX_SEL_UNUSABLE;
3527	}
3528
3529	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3530	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3531	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3532
3533	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3534	AssertRCReturn(rc, rc);
3535	return rc;
3536	}
3537
3538
3539	/**
3540	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3541	* into the guest-state area in the VMCS.
3542	*
3543	* @returns VBox status code.
3544	* @param pVM Pointer to the VM.
3545	* @param pVCPU Pointer to the VMCPU.
3546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3547	* out-of-sync. Make sure to update the required fields
3548	* before using them.
3549	*
3550	* @remarks Requires CR0 (strict builds validation).
3551	* @remarks No-long-jump zone!!!
3552	*/
3553	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3554	{
3555	int rc = VERR_INTERNAL_ERROR_5;
3556	PVM pVM = pVCpu->CTX_SUFF(pVM);
3557
3558	/*
3559	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3560	*/
3561	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3562	{
3563	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3564	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3565	{
3566	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3567	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3568	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3569	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3570	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3571	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3572	}
3573
3574	#ifdef VBOX_WITH_REM
3575	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3576	{
3577	Assert(pVM->hm.s.vmx.pRealModeTSS);
3578	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3579	if ( pVCpu->hm.s.vmx.fWasInRealMode
3580	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3581	{
3582	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3583	in real-mode (e.g. OpenBSD 4.0) */
3584	REMFlushTBs(pVM);
3585	Log4(("Load: Switch to protected mode detected!\n"));
3586	pVCpu->hm.s.vmx.fWasInRealMode = false;
3587	}
3588	}
3589	#endif
3590	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3591	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3592	AssertRCReturn(rc, rc);
3593	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3594	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3595	AssertRCReturn(rc, rc);
3596	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3597	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3598	AssertRCReturn(rc, rc);
3599	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3600	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3601	AssertRCReturn(rc, rc);
3602	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3603	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3604	AssertRCReturn(rc, rc);
3605	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3606	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3607	AssertRCReturn(rc, rc);
3608
3609	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3610	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3611	#ifdef VBOX_STRICT
3612	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3613	#endif
3614	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3615	}
3616
3617	/*
3618	* Guest TR.
3619	*/
3620	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3621	{
3622	/*
3623	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3624	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3625	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3626	*/
3627	uint16_t u16Sel = 0;
3628	uint32_t u32Limit = 0;
3629	uint64_t u64Base = 0;
3630	uint32_t u32AccessRights = 0;
3631
3632	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633	{
3634	u16Sel = pMixedCtx->tr.Sel;
3635	u32Limit = pMixedCtx->tr.u32Limit;
3636	u64Base = pMixedCtx->tr.u64Base;
3637	u32AccessRights = pMixedCtx->tr.Attr.u;
3638	}
3639	else
3640	{
3641	Assert(pVM->hm.s.vmx.pRealModeTSS);
3642	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3643
3644	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3645	RTGCPHYS GCPhys;
3646	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3647	AssertRCReturn(rc, rc);
3648
3649	X86DESCATTR DescAttr;
3650	DescAttr.u = 0;
3651	DescAttr.n.u1Present = 1;
3652	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3653
3654	u16Sel = 0;
3655	u32Limit = HM_VTX_TSS_SIZE;
3656	u64Base = GCPhys; /* in real-mode phys = virt. */
3657	u32AccessRights = DescAttr.u;
3658	}
3659
3660	/* Validate. */
3661	Assert(!(u16Sel & RT_BIT(2)));
3662	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3663	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3664	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3665	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3666	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3667	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3668	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3669	Assert( (u32Limit & 0xfff) == 0xfff
3670	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3671	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3672	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3673
3674	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3675	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3676	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3677	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3678
3679	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3680	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3681	}
3682
3683	/*
3684	* Guest GDTR.
3685	*/
3686	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3687	{
3688	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3689	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3690
3691	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3692	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3693	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3694	}
3695
3696	/*
3697	* Guest LDTR.
3698	*/
3699	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3700	{
3701	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3702	uint32_t u32Access = 0;
3703	if (!pMixedCtx->ldtr.Attr.u)
3704	u32Access = HMVMX_SEL_UNUSABLE;
3705	else
3706	u32Access = pMixedCtx->ldtr.Attr.u;
3707
3708	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3709	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3710	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3711	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3712
3713	/* Validate. */
3714	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3715	{
3716	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3717	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3718	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3719	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3720	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3721	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3722	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3723	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3724	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3725	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3726	}
3727
3728	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3729	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3730	}
3731
3732	/*
3733	* Guest IDTR.
3734	*/
3735	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3736	{
3737	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3738	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3739
3740	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3741	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3742	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3743	}
3744
3745	return VINF_SUCCESS;
3746	}
3747
3748
3749	/**
3750	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3751	* areas. These MSRs will automatically be loaded to the host CPU on every
3752	* successful VM entry and stored from the host CPU on every successful VM exit.
3753	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3754	*
3755	* @returns VBox status code.
3756	* @param pVCpu Pointer to the VMCPU.
3757	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3758	* out-of-sync. Make sure to update the required fields
3759	* before using them.
3760	*
3761	* @remarks No-long-jump zone!!!
3762	*/
3763	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3764	{
3765	AssertPtr(pVCpu);
3766	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3767
3768	/*
3769	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3770	*/
3771	int rc = VINF_SUCCESS;
3772	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3773	{
3774	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3775	PVM pVM = pVCpu->CTX_SUFF(pVM);
3776	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3777	uint32_t cGuestMsrs = 0;
3778
3779	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3780	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3781	* when the guest really is in 64-bit mode. */
3782	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3783	if (fSupportsLongMode)
3784	{
3785	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3786	pGuestMsr->u32Reserved = 0;
3787	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3788	pGuestMsr++; cGuestMsrs++;
3789	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3790	pGuestMsr->u32Reserved = 0;
3791	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3792	pGuestMsr++; cGuestMsrs++;
3793	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3794	pGuestMsr->u32Reserved = 0;
3795	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3796	pGuestMsr++; cGuestMsrs++;
3797	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3798	pGuestMsr->u32Reserved = 0;
3799	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3800	pGuestMsr++; cGuestMsrs++;
3801	}
3802
3803	/*
3804	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3805	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3806	*/
3807	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3808	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3809	{
3810	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3811	pGuestMsr->u32Reserved = 0;
3812	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3813	AssertRCReturn(rc, rc);
3814	pGuestMsr++; cGuestMsrs++;
3815	}
3816
3817	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3818	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3819	{
3820	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3821	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3822	}
3823
3824	/* Update the VCPU's copy of the guest MSR count. */
3825	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3826	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3827	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3828	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3829
3830	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3831	}
3832
3833	/*
3834	* Guest Sysenter MSRs.
3835	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3836	* VM-exits on WRMSRs for these MSRs.
3837	*/
3838	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3839	{
3840	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3841	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3842	}
3843	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3844	{
3845	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3846	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3847	}
3848	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3849	{
3850	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3851	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3852	}
3853
3854	return rc;
3855	}
3856
3857
3858	/**
3859	* Loads the guest activity state into the guest-state area in the VMCS.
3860	*
3861	* @returns VBox status code.
3862	* @param pVCpu Pointer to the VMCPU.
3863	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3864	* out-of-sync. Make sure to update the required fields
3865	* before using them.
3866	*
3867	* @remarks No-long-jump zone!!!
3868	*/
3869	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3870	{
3871	/** @todo See if we can make use of other states, e.g.
3872	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3873	int rc = VINF_SUCCESS;
3874	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3875	{
3876	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3877	AssertRCReturn(rc, rc);
3878	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3879	}
3880	return rc;
3881	}
3882
3883
3884	/**
3885	* Sets up the appropriate function to run guest code.
3886	*
3887	* @returns VBox status code.
3888	* @param pVCpu Pointer to the VMCPU.
3889	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3890	* out-of-sync. Make sure to update the required fields
3891	* before using them.
3892	*
3893	* @remarks No-long-jump zone!!!
3894	*/
3895	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3896	{
3897	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3898	{
3899	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3900	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3901	#endif
3902	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3903	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3904	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3905	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3906	#else
3907	/* 64-bit host or hybrid host. */
3908	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3909	#endif
3910	}
3911	else
3912	{
3913	/* Guest is not in long mode, use the 32-bit handler. */
3914	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3915	}
3916	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3917	return VINF_SUCCESS;
3918	}
3919
3920
3921	/**
3922	* Wrapper for running the guest code in VT-x.
3923	*
3924	* @returns VBox strict status code.
3925	* @param pVM Pointer to the VM.
3926	* @param pVCpu Pointer to the VMCPU.
3927	* @param pCtx Pointer to the guest-CPU context.
3928	*
3929	* @remarks No-long-jump zone!!!
3930	*/
3931	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3932	{
3933	/*
3934	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3935	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3936	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3937	*/
3938	#ifdef VBOX_WITH_KERNEL_USING_XMM
3939	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3940	#else
3941	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3942	#endif
3943	}
3944
3945
3946	/**
3947	* Report world-switch error and dump some useful debug info.
3948	*
3949	* @param pVM Pointer to the VM.
3950	* @param pVCpu Pointer to the VMCPU.
3951	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3952	* @param pCtx Pointer to the guest-CPU context.
3953	* @param pVmxTransient Pointer to the VMX transient structure (only
3954	* exitReason updated).
3955	*/
3956	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3957	{
3958	Assert(pVM);
3959	Assert(pVCpu);
3960	Assert(pCtx);
3961	Assert(pVmxTransient);
3962	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3963
3964	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3965	switch (rcVMRun)
3966	{
3967	case VERR_VMX_INVALID_VMXON_PTR:
3968	AssertFailed();
3969	break;
3970	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3971	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3972	{
3973	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3974	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3975	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3976	AssertRC(rc);
3977
3978	#ifdef VBOX_STRICT
3979	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3980	pVmxTransient->uExitReason));
3981	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3982	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3983	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3984	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3985	else
3986	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3987
3988	/* VMX control bits. */
3989	uint32_t u32Val;
3990	uint64_t u64Val;
3991	HMVMXHCUINTREG uHCReg;
3992	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3993	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3994	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3995	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3996	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3997	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3998	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3999	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4000	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4001	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4002	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4003	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4004	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4005	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4006	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4007	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4008	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4009	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4010	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4011	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4012	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4013	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4014	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4015	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4016	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4017	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4018	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4019	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4020	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4021	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4022	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4023	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4024	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4025	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4026	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4027	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4028	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4029	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4030	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4031	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4032	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4033	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4034
4035	/* Guest bits. */
4036	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4037	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4038	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4039	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4040	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4041	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4042	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4043	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4044
4045	/* Host bits. */
4046	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4047	Log4(("Host CR0 %#RHr\n", uHCReg));
4048	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4049	Log4(("Host CR3 %#RHr\n", uHCReg));
4050	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4051	Log4(("Host CR4 %#RHr\n", uHCReg));
4052
4053	RTGDTR HostGdtr;
4054	PCX86DESCHC pDesc;
4055	ASMGetGDTR(&HostGdtr);
4056	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4057	Log4(("Host CS %#08x\n", u32Val));
4058	if (u32Val < HostGdtr.cbGdt)
4059	{
4060	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4061	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4062	}
4063
4064	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4065	Log4(("Host DS %#08x\n", u32Val));
4066	if (u32Val < HostGdtr.cbGdt)
4067	{
4068	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4069	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4070	}
4071
4072	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4073	Log4(("Host ES %#08x\n", u32Val));
4074	if (u32Val < HostGdtr.cbGdt)
4075	{
4076	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4077	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4078	}
4079
4080	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4081	Log4(("Host FS %#08x\n", u32Val));
4082	if (u32Val < HostGdtr.cbGdt)
4083	{
4084	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4085	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4086	}
4087
4088	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4089	Log4(("Host GS %#08x\n", u32Val));
4090	if (u32Val < HostGdtr.cbGdt)
4091	{
4092	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4093	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4094	}
4095
4096	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4097	Log4(("Host SS %#08x\n", u32Val));
4098	if (u32Val < HostGdtr.cbGdt)
4099	{
4100	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4101	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4102	}
4103
4104	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4105	Log4(("Host TR %#08x\n", u32Val));
4106	if (u32Val < HostGdtr.cbGdt)
4107	{
4108	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4109	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4110	}
4111
4112	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4113	Log4(("Host TR Base %#RHv\n", uHCReg));
4114	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4115	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4116	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4117	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4118	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4119	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4120	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4121	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4122	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4123	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4124	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4125	Log4(("Host RSP %#RHv\n", uHCReg));
4126	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4127	Log4(("Host RIP %#RHv\n", uHCReg));
4128	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4129	if (HMVMX_IS_64BIT_HOST_MODE())
4130	{
4131	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4132	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4133	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4134	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4135	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4136	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4137	}
4138	# endif
4139	#endif /* VBOX_STRICT */
4140	break;
4141	}
4142
4143	default:
4144	/* Impossible */
4145	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4146	break;
4147	}
4148	NOREF(pVM);
4149	}
4150
4151
4152	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4153	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4154	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4155	#endif
4156	#ifdef VBOX_STRICT
4157	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4158	{
4159	switch (idxField)
4160	{
4161	case VMX_VMCS_GUEST_RIP:
4162	case VMX_VMCS_GUEST_RSP:
4163	case VMX_VMCS_GUEST_SYSENTER_EIP:
4164	case VMX_VMCS_GUEST_SYSENTER_ESP:
4165	case VMX_VMCS_GUEST_GDTR_BASE:
4166	case VMX_VMCS_GUEST_IDTR_BASE:
4167	case VMX_VMCS_GUEST_CS_BASE:
4168	case VMX_VMCS_GUEST_DS_BASE:
4169	case VMX_VMCS_GUEST_ES_BASE:
4170	case VMX_VMCS_GUEST_FS_BASE:
4171	case VMX_VMCS_GUEST_GS_BASE:
4172	case VMX_VMCS_GUEST_SS_BASE:
4173	case VMX_VMCS_GUEST_LDTR_BASE:
4174	case VMX_VMCS_GUEST_TR_BASE:
4175	case VMX_VMCS_GUEST_CR3:
4176	return true;
4177	}
4178	return false;
4179	}
4180
4181	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4182	{
4183	switch (idxField)
4184	{
4185	/* Read-only fields. */
4186	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4187	return true;
4188	}
4189	/* Remaining readable fields should also be writable. */
4190	return hmR0VmxIsValidWriteField(idxField);
4191	}
4192	#endif /* VBOX_STRICT */
4193
4194
4195	/**
4196	* Executes the specified handler in 64-bit mode.
4197	*
4198	* @returns VBox status code.
4199	* @param pVM Pointer to the VM.
4200	* @param pVCpu Pointer to the VMCPU.
4201	* @param pCtx Pointer to the guest CPU context.
4202	* @param enmOp The operation to perform.
4203	* @param cbParam Number of parameters.
4204	* @param paParam Array of 32-bit parameters.
4205	*/
4206	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4207	uint32_t *paParam)
4208	{
4209	int rc, rc2;
4210	PHMGLOBLCPUINFO pCpu;
4211	RTHCPHYS HCPhysCpuPage;
4212	RTCCUINTREG uOldEFlags;
4213
4214	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4215	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4216	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4217	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4218
4219	#ifdef VBOX_STRICT
4220	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4221	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4222
4223	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4224	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4225	#endif
4226
4227	/* Disable interrupts. */
4228	uOldEFlags = ASMIntDisableFlags();
4229
4230	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4231	RTCPUID idHostCpu = RTMpCpuId();
4232	CPUMR0SetLApic(pVM, idHostCpu);
4233	#endif
4234
4235	pCpu = HMR0GetCurrentCpu();
4236	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4237
4238	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4239	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4240
4241	/* Leave VMX Root Mode. */
4242	VMXDisable();
4243
4244	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4245
4246	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4247	CPUMSetHyperEIP(pVCpu, enmOp);
4248	for (int i = (int)cbParam - 1; i >= 0; i--)
4249	CPUMPushHyper(pVCpu, paParam[i]);
4250
4251	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4252
4253	/* Call the switcher. */
4254	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4255	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4256
4257	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4258	/* Make sure the VMX instructions don't cause #UD faults. */
4259	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4260
4261	/* Re-enter VMX Root Mode */
4262	rc2 = VMXEnable(HCPhysCpuPage);
4263	if (RT_FAILURE(rc2))
4264	{
4265	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4266	ASMSetFlags(uOldEFlags);
4267	return rc2;
4268	}
4269
4270	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4271	AssertRC(rc2);
4272	Assert(!(ASMGetFlags() & X86_EFL_IF));
4273	ASMSetFlags(uOldEFlags);
4274	return rc;
4275	}
4276
4277
4278	/**
4279	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4280	* supporting 64-bit guests.
4281	*
4282	* @returns VBox status code.
4283	* @param fResume Whether to VMLAUNCH or VMRESUME.
4284	* @param pCtx Pointer to the guest-CPU context.
4285	* @param pCache Pointer to the VMCS cache.
4286	* @param pVM Pointer to the VM.
4287	* @param pVCpu Pointer to the VMCPU.
4288	*/
4289	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4290	{
4291	uint32_t aParam[6];
4292	PHMGLOBLCPUINFO pCpu = NULL;
4293	RTHCPHYS HCPhysCpuPage = 0;
4294	int rc = VERR_INTERNAL_ERROR_5;
4295
4296	pCpu = HMR0GetCurrentCpu();
4297	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4298
4299	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4300	pCache->uPos = 1;
4301	pCache->interPD = PGMGetInterPaeCR3(pVM);
4302	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4303	#endif
4304
4305	#ifdef VBOX_STRICT
4306	pCache->TestIn.HCPhysCpuPage = 0;
4307	pCache->TestIn.HCPhysVmcs = 0;
4308	pCache->TestIn.pCache = 0;
4309	pCache->TestOut.HCPhysVmcs = 0;
4310	pCache->TestOut.pCache = 0;
4311	pCache->TestOut.pCtx = 0;
4312	pCache->TestOut.eflags = 0;
4313	#endif
4314
4315	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4316	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4317	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4318	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4319	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4320	aParam[5] = 0;
4321
4322	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4323	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4324	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4325	#endif
4326	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4327
4328	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4329	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4330	Assert(pCtx->dr[4] == 10);
4331	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4332	#endif
4333
4334	#ifdef VBOX_STRICT
4335	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4336	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4337	pVCpu->hm.s.vmx.HCPhysVmcs));
4338	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4339	pCache->TestOut.HCPhysVmcs));
4340	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4341	pCache->TestOut.pCache));
4342	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4343	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4344	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4345	pCache->TestOut.pCtx));
4346	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4347	#endif
4348	return rc;
4349	}
4350
4351
4352	/**
4353	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4354	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4355	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4356	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4357	*
4358	* @returns VBox status code.
4359	* @param pVM Pointer to the VM.
4360	* @param pVCpu Pointer to the VMCPU.
4361	*/
4362	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4363	{
4364	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4365	{ \
4366	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4367	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4368	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4369	++cReadFields; \
4370	}
4371
4372	AssertPtr(pVM);
4373	AssertPtr(pVCpu);
4374	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4375	uint32_t cReadFields = 0;
4376
4377	/*
4378	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4379	* and serve to indicate exceptions to the rules.
4380	*/
4381
4382	/* Guest-natural selector base fields. */
4383	#if 0
4384	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4385	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4387	#endif
4388	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4390	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4392	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4393	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4396	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4397	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4398	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4399	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4400	#if 0
4401	/* Unused natural width guest-state fields. */
4402	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4404	#endif
4405	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4406	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4407
4408	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4409	#if 0
4410	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4412	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4413	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4414	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4415	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4416	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4417	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4418	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4419	#endif
4420
4421	/* Natural width guest-state fields. */
4422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4423	#if 0
4424	/* Currently unused field. */
4425	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4426	#endif
4427
4428	if (pVM->hm.s.fNestedPaging)
4429	{
4430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4431	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4432	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4433	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4434	}
4435	else
4436	{
4437	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4438	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4439	}
4440
4441	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4442	return VINF_SUCCESS;
4443	}
4444
4445
4446	/**
4447	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4448	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4449	* darwin, running 64-bit guests).
4450	*
4451	* @returns VBox status code.
4452	* @param pVCpu Pointer to the VMCPU.
4453	* @param idxField The VMCS field encoding.
4454	* @param u64Val 16, 32 or 64 bits value.
4455	*/
4456	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4457	{
4458	int rc;
4459	switch (idxField)
4460	{
4461	/*
4462	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4463	*/
4464	/* 64-bit Control fields. */
4465	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4466	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4467	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4468	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4469	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4470	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4471	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4472	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4473	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4474	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4475	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4476	case VMX_VMCS64_CTRL_EPTP_FULL:
4477	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4478	/* 64-bit Guest-state fields. */
4479	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4480	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4481	case VMX_VMCS64_GUEST_PAT_FULL:
4482	case VMX_VMCS64_GUEST_EFER_FULL:
4483	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4484	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4485	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4486	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4487	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4488	/* 64-bit Host-state fields. */
4489	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4490	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4491	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4492	{
4493	rc = VMXWriteVmcs32(idxField, u64Val);
4494	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4495	break;
4496	}
4497
4498	/*
4499	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4500	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4501	*/
4502	/* Natural-width Guest-state fields. */
4503	case VMX_VMCS_GUEST_CR3:
4504	case VMX_VMCS_GUEST_ES_BASE:
4505	case VMX_VMCS_GUEST_CS_BASE:
4506	case VMX_VMCS_GUEST_SS_BASE:
4507	case VMX_VMCS_GUEST_DS_BASE:
4508	case VMX_VMCS_GUEST_FS_BASE:
4509	case VMX_VMCS_GUEST_GS_BASE:
4510	case VMX_VMCS_GUEST_LDTR_BASE:
4511	case VMX_VMCS_GUEST_TR_BASE:
4512	case VMX_VMCS_GUEST_GDTR_BASE:
4513	case VMX_VMCS_GUEST_IDTR_BASE:
4514	case VMX_VMCS_GUEST_RSP:
4515	case VMX_VMCS_GUEST_RIP:
4516	case VMX_VMCS_GUEST_SYSENTER_ESP:
4517	case VMX_VMCS_GUEST_SYSENTER_EIP:
4518	{
4519	if (!(u64Val >> 32))
4520	{
4521	/* If this field is 64-bit, VT-x will zero out the top bits. */
4522	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4523	}
4524	else
4525	{
4526	/* Assert that only the 32->64 switcher case should ever come here. */
4527	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4528	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4529	}
4530	break;
4531	}
4532
4533	default:
4534	{
4535	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4536	rc = VERR_INVALID_PARAMETER;
4537	break;
4538	}
4539	}
4540	AssertRCReturn(rc, rc);
4541	return rc;
4542	}
4543
4544
4545	/**
4546	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4547	* hosts (except darwin) for 64-bit guests.
4548	*
4549	* @param pVCpu Pointer to the VMCPU.
4550	* @param idxField The VMCS field encoding.
4551	* @param u64Val 16, 32 or 64 bits value.
4552	*/
4553	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4554	{
4555	AssertPtr(pVCpu);
4556	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4557
4558	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4559	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4560
4561	/* Make sure there are no duplicates. */
4562	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4563	{
4564	if (pCache->Write.aField[i] == idxField)
4565	{
4566	pCache->Write.aFieldVal[i] = u64Val;
4567	return VINF_SUCCESS;
4568	}
4569	}
4570
4571	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4572	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4573	pCache->Write.cValidEntries++;
4574	return VINF_SUCCESS;
4575	}
4576
4577	/* Enable later when the assembly code uses these as callbacks. */
4578	#if 0
4579	/*
4580	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4581	*
4582	* @param pVCpu Pointer to the VMCPU.
4583	* @param pCache Pointer to the VMCS cache.
4584	*
4585	* @remarks No-long-jump zone!!!
4586	*/
4587	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4588	{
4589	AssertPtr(pCache);
4590	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4591	{
4592	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4593	AssertRC(rc);
4594	}
4595	pCache->Write.cValidEntries = 0;
4596	}
4597
4598
4599	/**
4600	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4601	*
4602	* @param pVCpu Pointer to the VMCPU.
4603	* @param pCache Pointer to the VMCS cache.
4604	*
4605	* @remarks No-long-jump zone!!!
4606	*/
4607	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4608	{
4609	AssertPtr(pCache);
4610	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4611	{
4612	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4613	AssertRC(rc);
4614	}
4615	}
4616	#endif
4617	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4618
4619
4620	/**
4621	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4622	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4623	* timer.
4624	*
4625	* @returns VBox status code.
4626	* @param pVCpu Pointer to the VMCPU.
4627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4628	* out-of-sync. Make sure to update the required fields
4629	* before using them.
4630	* @remarks No-long-jump zone!!!
4631	*/
4632	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4633	{
4634	int rc = VERR_INTERNAL_ERROR_5;
4635	bool fOffsettedTsc = false;
4636	PVM pVM = pVCpu->CTX_SUFF(pVM);
4637	if (pVM->hm.s.vmx.fUsePreemptTimer)
4638	{
4639	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4640
4641	/* Make sure the returned values have sane upper and lower boundaries. */
4642	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4643	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4644	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4645	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4646
4647	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4648	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4649	}
4650	else
4651	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4652
4653	if (fOffsettedTsc)
4654	{
4655	uint64_t u64CurTSC = ASMReadTSC();
4656	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4657	{
4658	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4659	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4660
4661	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4662	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4663	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4664	}
4665	else
4666	{
4667	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4668	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4669	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4670	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4671	}
4672	}
4673	else
4674	{
4675	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4676	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4677	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4678	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4679	}
4680	}
4681
4682
4683	/**
4684	* Determines if an exception is a contributory exception. Contributory
4685	* exceptions are ones which can cause double-faults. Page-fault is
4686	* intentionally not included here as it's a conditional contributory exception.
4687	*
4688	* @returns true if the exception is contributory, false otherwise.
4689	* @param uVector The exception vector.
4690	*/
4691	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4692	{
4693	switch (uVector)
4694	{
4695	case X86_XCPT_GP:
4696	case X86_XCPT_SS:
4697	case X86_XCPT_NP:
4698	case X86_XCPT_TS:
4699	case X86_XCPT_DE:
4700	return true;
4701	default:
4702	break;
4703	}
4704	return false;
4705	}
4706
4707
4708	/**
4709	* Sets an event as a pending event to be injected into the guest.
4710	*
4711	* @param pVCpu Pointer to the VMCPU.
4712	* @param u32IntrInfo The VM-entry interruption-information field.
4713	* @param cbInstr The VM-entry instruction length in bytes (for software
4714	* interrupts, exceptions and privileged software
4715	* exceptions).
4716	* @param u32ErrCode The VM-entry exception error code.
4717	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4718	* page-fault.
4719	*/
4720	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4721	RTGCUINTPTR GCPtrFaultAddress)
4722	{
4723	Assert(!pVCpu->hm.s.Event.fPending);
4724	pVCpu->hm.s.Event.fPending = true;
4725	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4726	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4727	pVCpu->hm.s.Event.cbInstr = cbInstr;
4728	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4729	}
4730
4731
4732	/**
4733	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4734	*
4735	* @param pVCpu Pointer to the VMCPU.
4736	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4737	* out-of-sync. Make sure to update the required fields
4738	* before using them.
4739	*/
4740	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4741	{
4742	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4743	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4744	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4745	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4746	}
4747
4748
4749	/**
4750	* Handle a condition that occurred while delivering an event through the guest
4751	* IDT.
4752	*
4753	* @returns VBox status code (informational error codes included).
4754	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4755	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4756	* continue execution of the guest which will delivery the #DF.
4757	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4758	*
4759	* @param pVCpu Pointer to the VMCPU.
4760	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4761	* out-of-sync. Make sure to update the required fields
4762	* before using them.
4763	* @param pVmxTransient Pointer to the VMX transient structure.
4764	*
4765	* @remarks No-long-jump zone!!!
4766	*/
4767	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4768	{
4769	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4770	AssertRC(rc);
4771	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4772	{
4773	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4774	AssertRCReturn(rc, rc);
4775
4776	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4777	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4778	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4779
4780	typedef enum
4781	{
4782	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4783	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4784	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4785	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4786	} VMXREFLECTXCPT;
4787
4788	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4789	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4790	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4791	{
4792	enmReflect = VMXREFLECTXCPT_XCPT;
4793	#ifdef VBOX_STRICT
4794	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4795	&& uExitVector == X86_XCPT_PF)
4796	{
4797	Log4(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4798	}
4799	#endif
4800	if ( uExitVector == X86_XCPT_PF
4801	&& uIdtVector == X86_XCPT_PF)
4802	{
4803	pVmxTransient->fVectoringPF = true;
4804	Log4(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4805	}
4806	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4807	&& hmR0VmxIsContributoryXcpt(uExitVector)
4808	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4809	\|\| uIdtVector == X86_XCPT_PF))
4810	{
4811	enmReflect = VMXREFLECTXCPT_DF;
4812	}
4813	else if (uIdtVector == X86_XCPT_DF)
4814	enmReflect = VMXREFLECTXCPT_TF;
4815	}
4816	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4817	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4818	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4819	{
4820	/*
4821	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4822	* (whatever they are) as they reoccur when restarting the instruction.
4823	*/
4824	enmReflect = VMXREFLECTXCPT_XCPT;
4825	}
4826
4827	switch (enmReflect)
4828	{
4829	case VMXREFLECTXCPT_XCPT:
4830	{
4831	uint32_t u32ErrCode = 0;
4832	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4833	{
4834	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4835	AssertRCReturn(rc, rc);
4836	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4837	}
4838
4839	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4840	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4841	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4842	rc = VINF_SUCCESS;
4843	Log4(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4844	pVCpu->hm.s.Event.u32ErrCode));
4845	break;
4846	}
4847
4848	case VMXREFLECTXCPT_DF:
4849	{
4850	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4851	rc = VINF_VMX_DOUBLE_FAULT;
4852	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4853	uIdtVector, uExitVector));
4854	break;
4855	}
4856
4857	case VMXREFLECTXCPT_TF:
4858	{
4859	Log4(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4860	rc = VINF_EM_RESET;
4861	break;
4862	}
4863
4864	default:
4865	Assert(rc == VINF_SUCCESS);
4866	break;
4867	}
4868	}
4869	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4870	return rc;
4871	}
4872
4873
4874	/**
4875	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4876	*
4877	* @returns VBox status code.
4878	* @param pVCpu Pointer to the VMCPU.
4879	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4880	* out-of-sync. Make sure to update the required fields
4881	* before using them.
4882	*
4883	* @remarks No-long-jump zone!!!
4884	*/
4885	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4886	{
4887	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4888	{
4889	uint32_t uVal = 0;
4890	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4891	AssertRCReturn(rc, rc);
4892	uint32_t uShadow = 0;
4893	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4894	AssertRCReturn(rc, rc);
4895
4896	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4897	CPUMSetGuestCR0(pVCpu, uVal);
4898	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4899	}
4900	return VINF_SUCCESS;
4901	}
4902
4903
4904	/**
4905	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4906	*
4907	* @returns VBox status code.
4908	* @param pVCpu Pointer to the VMCPU.
4909	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4910	* out-of-sync. Make sure to update the required fields
4911	* before using them.
4912	*
4913	* @remarks No-long-jump zone!!!
4914	*/
4915	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4916	{
4917	int rc = VINF_SUCCESS;
4918	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4919	{
4920	uint32_t uVal = 0;
4921	uint32_t uShadow = 0;
4922	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4923	AssertRCReturn(rc, rc);
4924	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4925	AssertRCReturn(rc, rc);
4926
4927	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4928	CPUMSetGuestCR4(pVCpu, uVal);
4929	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4930	}
4931	return rc;
4932	}
4933
4934
4935	/**
4936	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4937	*
4938	* @returns VBox status code.
4939	* @param pVCpu Pointer to the VMCPU.
4940	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4941	* out-of-sync. Make sure to update the required fields
4942	* before using them.
4943	*
4944	* @remarks No-long-jump zone!!!
4945	*/
4946	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4947	{
4948	int rc = VINF_SUCCESS;
4949	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4950	{
4951	uint64_t u64Val = 0;
4952	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4953	AssertRCReturn(rc, rc);
4954
4955	pMixedCtx->rip = u64Val;
4956	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4957	}
4958	return rc;
4959	}
4960
4961
4962	/**
4963	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4964	*
4965	* @returns VBox status code.
4966	* @param pVCpu Pointer to the VMCPU.
4967	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4968	* out-of-sync. Make sure to update the required fields
4969	* before using them.
4970	*
4971	* @remarks No-long-jump zone!!!
4972	*/
4973	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4974	{
4975	int rc = VINF_SUCCESS;
4976	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4977	{
4978	uint64_t u64Val = 0;
4979	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4980	AssertRCReturn(rc, rc);
4981
4982	pMixedCtx->rsp = u64Val;
4983	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4984	}
4985	return rc;
4986	}
4987
4988
4989	/**
4990	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4991	*
4992	* @returns VBox status code.
4993	* @param pVCpu Pointer to the VMCPU.
4994	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4995	* out-of-sync. Make sure to update the required fields
4996	* before using them.
4997	*
4998	* @remarks No-long-jump zone!!!
4999	*/
5000	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5001	{
5002	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5003	{
5004	uint32_t uVal = 0;
5005	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5006	AssertRCReturn(rc, rc);
5007
5008	pMixedCtx->eflags.u32 = uVal;
5009	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5010	{
5011	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5012	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5013
5014	pMixedCtx->eflags.Bits.u1VM = 0;
5015	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5016	}
5017
5018	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5019	}
5020	return VINF_SUCCESS;
5021	}
5022
5023
5024	/**
5025	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5026	* guest-CPU context.
5027	*/
5028	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5029	{
5030	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5031	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5032	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5033	return rc;
5034	}
5035
5036
5037	/**
5038	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5039	* from the guest-state area in the VMCS.
5040	*
5041	* @param pVCpu Pointer to the VMCPU.
5042	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5043	* out-of-sync. Make sure to update the required fields
5044	* before using them.
5045	*
5046	* @remarks No-long-jump zone!!!
5047	*/
5048	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5049	{
5050	uint32_t uIntrState = 0;
5051	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5052	AssertRC(rc);
5053
5054	if (!uIntrState)
5055	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5056	else
5057	{
5058	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5059	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5060	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5061	AssertRC(rc);
5062	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5063	AssertRC(rc);
5064
5065	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5066	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5067	}
5068	}
5069
5070
5071	/**
5072	* Saves the guest's activity state.
5073	*
5074	* @returns VBox status code.
5075	* @param pVCpu Pointer to the VMCPU.
5076	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5077	* out-of-sync. Make sure to update the required fields
5078	* before using them.
5079	*
5080	* @remarks No-long-jump zone!!!
5081	*/
5082	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5083	{
5084	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5085	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5086	return VINF_SUCCESS;
5087	}
5088
5089
5090	/**
5091	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5092	* the current VMCS into the guest-CPU context.
5093	*
5094	* @returns VBox status code.
5095	* @param pVCpu Pointer to the VMCPU.
5096	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5097	* out-of-sync. Make sure to update the required fields
5098	* before using them.
5099	*
5100	* @remarks No-long-jump zone!!!
5101	*/
5102	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5103	{
5104	int rc = VINF_SUCCESS;
5105	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5106	{
5107	uint32_t u32Val = 0;
5108	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5109	pMixedCtx->SysEnter.cs = u32Val;
5110	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5111	}
5112
5113	uint64_t u64Val = 0;
5114	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5115	{
5116	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5117	pMixedCtx->SysEnter.eip = u64Val;
5118	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5119	}
5120	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5121	{
5122	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5123	pMixedCtx->SysEnter.esp = u64Val;
5124	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5125	}
5126	return rc;
5127	}
5128
5129
5130	/**
5131	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5132	* context.
5133	*
5134	* @returns VBox status code.
5135	* @param pVCpu Pointer to the VMCPU.
5136	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5137	* out-of-sync. Make sure to update the required fields
5138	* before using them.
5139	*
5140	* @remarks No-long-jump zone!!!
5141	*/
5142	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5143	{
5144	int rc = VINF_SUCCESS;
5145	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5146	{
5147	uint64_t u64Val = 0;
5148	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5149	pMixedCtx->fs.u64Base = u64Val;
5150	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5151	}
5152	return rc;
5153	}
5154
5155
5156	/**
5157	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5158	* context.
5159	*
5160	* @returns VBox status code.
5161	* @param pVCpu Pointer to the VMCPU.
5162	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5163	* out-of-sync. Make sure to update the required fields
5164	* before using them.
5165	*
5166	* @remarks No-long-jump zone!!!
5167	*/
5168	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5169	{
5170	int rc = VINF_SUCCESS;
5171	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5172	{
5173	uint64_t u64Val = 0;
5174	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5175	pMixedCtx->gs.u64Base = u64Val;
5176	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5177	}
5178	return rc;
5179	}
5180
5181
5182	/**
5183	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5184	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5185	* and TSC_AUX.
5186	*
5187	* @returns VBox status code.
5188	* @param pVCpu Pointer to the VMCPU.
5189	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5190	* out-of-sync. Make sure to update the required fields
5191	* before using them.
5192	*
5193	* @remarks No-long-jump zone!!!
5194	*/
5195	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5196	{
5197	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5198	return VINF_SUCCESS;
5199
5200	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5201	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5202	{
5203	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5204	pMsr += i;
5205	switch (pMsr->u32IndexMSR)
5206	{
5207	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5208	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5209	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5210	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5211	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5212	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5213	default:
5214	{
5215	AssertFailed();
5216	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5217	}
5218	}
5219	}
5220	#endif
5221
5222	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5223	return VINF_SUCCESS;
5224	}
5225
5226
5227	/**
5228	* Saves the guest control registers from the current VMCS into the guest-CPU
5229	* context.
5230	*
5231	* @returns VBox status code.
5232	* @param pVCpu Pointer to the VMCPU.
5233	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5234	* out-of-sync. Make sure to update the required fields
5235	* before using them.
5236	*
5237	* @remarks No-long-jump zone!!!
5238	*/
5239	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5240	{
5241	/* Guest CR0. Guest FPU. */
5242	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5243	AssertRCReturn(rc, rc);
5244
5245	/* Guest CR4. */
5246	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5247	AssertRCReturn(rc, rc);
5248
5249	/* Guest CR2 - updated always during the world-switch or in #PF. */
5250	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5251	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5252	{
5253	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5254	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5255
5256	PVM pVM = pVCpu->CTX_SUFF(pVM);
5257	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5258	\|\| ( pVM->hm.s.fNestedPaging
5259	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5260	{
5261	uint64_t u64Val = 0;
5262	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5263	if (pMixedCtx->cr3 != u64Val)
5264	{
5265	CPUMSetGuestCR3(pVCpu, u64Val);
5266	if (VMMRZCallRing3IsEnabled(pVCpu))
5267	{
5268	PGMUpdateCR3(pVCpu, u64Val);
5269	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5270	}
5271	else
5272	{
5273	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5274	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5275	}
5276	}
5277
5278	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5279	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5280	{
5281	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5282	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5283	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5284	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5285
5286	if (VMMRZCallRing3IsEnabled(pVCpu))
5287	{
5288	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5289	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5290	}
5291	else
5292	{
5293	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5294	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5295	}
5296	}
5297	}
5298
5299	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5300	}
5301
5302	/*
5303	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5304	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5305	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5306	*
5307	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5308	*/
5309	if (VMMRZCallRing3IsEnabled(pVCpu))
5310	{
5311	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5312	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5313
5314	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5315	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5316
5317	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5318	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5319	}
5320
5321	return rc;
5322	}
5323
5324
5325	/**
5326	* Reads a guest segment register from the current VMCS into the guest-CPU
5327	* context.
5328	*
5329	* @returns VBox status code.
5330	* @param pVCpu Pointer to the VMCPU.
5331	* @param idxSel Index of the selector in the VMCS.
5332	* @param idxLimit Index of the segment limit in the VMCS.
5333	* @param idxBase Index of the segment base in the VMCS.
5334	* @param idxAccess Index of the access rights of the segment in the VMCS.
5335	* @param pSelReg Pointer to the segment selector.
5336	*
5337	* @remarks No-long-jump zone!!!
5338	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5339	* macro as that takes care of whether to read from the VMCS cache or
5340	* not.
5341	*/
5342	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5343	PCPUMSELREG pSelReg)
5344	{
5345	uint32_t u32Val = 0;
5346	int rc = VMXReadVmcs32(idxSel, &u32Val);
5347	AssertRCReturn(rc, rc);
5348	pSelReg->Sel = (uint16_t)u32Val;
5349	pSelReg->ValidSel = (uint16_t)u32Val;
5350	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5351
5352	rc = VMXReadVmcs32(idxLimit, &u32Val);
5353	AssertRCReturn(rc, rc);
5354	pSelReg->u32Limit = u32Val;
5355
5356	uint64_t u64Val = 0;
5357	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5358	AssertRCReturn(rc, rc);
5359	pSelReg->u64Base = u64Val;
5360
5361	rc = VMXReadVmcs32(idxAccess, &u32Val);
5362	AssertRCReturn(rc, rc);
5363	pSelReg->Attr.u = u32Val;
5364
5365	/*
5366	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5367	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5368	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5369	*
5370	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5371	*/
5372	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5373	{
5374	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5375	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5376	}
5377	return VINF_SUCCESS;
5378	}
5379
5380
5381	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5382	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5383	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5384	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5385	#else
5386	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5387	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5388	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5389	#endif
5390
5391
5392	/**
5393	* Saves the guest segment registers from the current VMCS into the guest-CPU
5394	* context.
5395	*
5396	* @returns VBox status code.
5397	* @param pVCpu Pointer to the VMCPU.
5398	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5399	* out-of-sync. Make sure to update the required fields
5400	* before using them.
5401	*
5402	* @remarks No-long-jump zone!!!
5403	*/
5404	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5405	{
5406	/* Guest segment registers. */
5407	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5408	{
5409	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5410	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5411	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5412	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5413	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5414	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5415	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5416
5417	/* Restore segment attributes for real-on-v86 mode hack. */
5418	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5419	{
5420	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5421	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5422	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5423	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5424	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5425	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5426	}
5427	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5428	}
5429
5430	return VINF_SUCCESS;
5431	}
5432
5433
5434	/**
5435	* Saves the guest descriptor table registers and task register from the current
5436	* VMCS into the guest-CPU context.
5437	*
5438	* @returns VBox status code.
5439	* @param pVCpu Pointer to the VMCPU.
5440	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5441	* out-of-sync. Make sure to update the required fields
5442	* before using them.
5443	*
5444	* @remarks No-long-jump zone!!!
5445	*/
5446	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5447	{
5448	int rc = VINF_SUCCESS;
5449
5450	/* Guest LDTR. */
5451	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5452	{
5453	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5454	AssertRCReturn(rc, rc);
5455	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5456	}
5457
5458	/* Guest GDTR. */
5459	uint64_t u64Val = 0;
5460	uint32_t u32Val = 0;
5461	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5462	{
5463	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5464	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5465	pMixedCtx->gdtr.pGdt = u64Val;
5466	pMixedCtx->gdtr.cbGdt = u32Val;
5467	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5468	}
5469
5470	/* Guest IDTR. */
5471	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5472	{
5473	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5474	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5475	pMixedCtx->idtr.pIdt = u64Val;
5476	pMixedCtx->idtr.cbIdt = u32Val;
5477	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5478	}
5479
5480	/* Guest TR. */
5481	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5482	{
5483	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5484	AssertRCReturn(rc, rc);
5485
5486	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5487	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5488	{
5489	rc = VMXLOCAL_READ_SEG(TR, tr);
5490	AssertRCReturn(rc, rc);
5491	}
5492	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5493	}
5494	return rc;
5495	}
5496
5497	#undef VMXLOCAL_READ_SEG
5498
5499
5500	/**
5501	* Saves the guest debug registers from the current VMCS into the guest-CPU
5502	* context.
5503	*
5504	* @returns VBox status code.
5505	* @param pVCpu Pointer to the VMCPU.
5506	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5507	* out-of-sync. Make sure to update the required fields
5508	* before using them.
5509	*
5510	* @remarks No-long-jump zone!!!
5511	*/
5512	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5513	{
5514	int rc = VINF_SUCCESS;
5515	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5516	{
5517	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5518	uint32_t u32Val;
5519	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5520	pMixedCtx->dr[7] = u32Val;
5521
5522	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5523	}
5524	return rc;
5525	}
5526
5527
5528	/**
5529	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5530	*
5531	* @returns VBox status code.
5532	* @param pVCpu Pointer to the VMCPU.
5533	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5534	* out-of-sync. Make sure to update the required fields
5535	* before using them.
5536	*
5537	* @remarks No-long-jump zone!!!
5538	*/
5539	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5540	{
5541	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5542	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5543	return VINF_SUCCESS;
5544	}
5545
5546
5547	/**
5548	* Saves the entire guest state from the currently active VMCS into the
5549	* guest-CPU context. This essentially VMREADs all guest-data.
5550	*
5551	* @returns VBox status code.
5552	* @param pVCpu Pointer to the VMCPU.
5553	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5554	* out-of-sync. Make sure to update the required fields
5555	* before using them.
5556	*/
5557	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5558	{
5559	Assert(pVCpu);
5560	Assert(pMixedCtx);
5561
5562	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5563	return VINF_SUCCESS;
5564
5565	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5566	there is no real need to. */
5567	if (VMMRZCallRing3IsEnabled(pVCpu))
5568	VMMR0LogFlushDisable(pVCpu);
5569	else
5570	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5571	Log4Func(("\n"));
5572
5573	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5574	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5575
5576	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5577	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5578
5579	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5580	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5581
5582	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5583	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5584
5585	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5586	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5587
5588	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5589	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5590
5591	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5592	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5593
5594	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5595	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5596
5597	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5598	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5599
5600	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5601	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5602
5603	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5604	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5605
5606	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5607	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5608
5609	if (VMMRZCallRing3IsEnabled(pVCpu))
5610	VMMR0LogFlushEnable(pVCpu);
5611
5612	return rc;
5613	}
5614
5615
5616	/**
5617	* Check per-VM and per-VCPU force flag actions that require us to go back to
5618	* ring-3 for one reason or another.
5619	*
5620	* @returns VBox status code (information status code included).
5621	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5622	* ring-3.
5623	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5624	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5625	* interrupts)
5626	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5627	* all EMTs to be in ring-3.
5628	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5629	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5630	* to the EM loop.
5631	*
5632	* @param pVM Pointer to the VM.
5633	* @param pVCpu Pointer to the VMCPU.
5634	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5635	* out-of-sync. Make sure to update the required fields
5636	* before using them.
5637	*/
5638	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5639	{
5640	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5641
5642	int rc = VERR_INTERNAL_ERROR_5;
5643	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5644	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5645	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5646	{
5647	/* We need the control registers now, make sure the guest-CPU context is updated. */
5648	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5649	AssertRCReturn(rc, rc);
5650
5651	/* Pending HM CR3 sync. */
5652	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5653	{
5654	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5655	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5656	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5657	}
5658
5659	/* Pending HM PAE PDPEs. */
5660	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5661	{
5662	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5663	AssertRC(rc);
5664	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5665	}
5666
5667	/* Pending PGM C3 sync. */
5668	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5669	{
5670	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5671	if (rc != VINF_SUCCESS)
5672	{
5673	AssertRC(rc);
5674	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5675	return rc;
5676	}
5677	}
5678
5679	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5680	/* -XXX- what was that about single stepping? */
5681	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5682	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5683	{
5684	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5685	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5686	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5687	return rc;
5688	}
5689
5690	/* Pending VM request packets, such as hardware interrupts. */
5691	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5692	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5693	{
5694	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5695	return VINF_EM_PENDING_REQUEST;
5696	}
5697
5698	/* Pending PGM pool flushes. */
5699	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5700	{
5701	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5702	return VINF_PGM_POOL_FLUSH_PENDING;
5703	}
5704
5705	/* Pending DMA requests. */
5706	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5707	{
5708	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5709	return VINF_EM_RAW_TO_R3;
5710	}
5711	}
5712
5713	/* Paranoia. */
5714	Assert(rc != VERR_EM_INTERPRETER);
5715	return VINF_SUCCESS;
5716	}
5717
5718
5719	/**
5720	* Converts any TRPM trap into a pending VMX event. This is typically used when
5721	* entering from ring-3 (not longjmp returns).
5722	*
5723	* @param pVCpu Pointer to the VMCPU.
5724	*/
5725	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5726	{
5727	Assert(TRPMHasTrap(pVCpu));
5728	Assert(!pVCpu->hm.s.Event.fPending);
5729
5730	uint8_t uVector;
5731	TRPMEVENT enmTrpmEvent;
5732	RTGCUINT uErrCode;
5733	RTGCUINTPTR GCPtrFaultAddress;
5734	uint8_t cbInstr;
5735
5736	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5737	AssertRC(rc);
5738
5739	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5740	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5741	if (enmTrpmEvent == TRPM_TRAP)
5742	{
5743	switch (uVector)
5744	{
5745	case X86_XCPT_BP:
5746	case X86_XCPT_OF:
5747	{
5748	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5749	break;
5750	}
5751
5752	case X86_XCPT_PF:
5753	case X86_XCPT_DF:
5754	case X86_XCPT_TS:
5755	case X86_XCPT_NP:
5756	case X86_XCPT_SS:
5757	case X86_XCPT_GP:
5758	case X86_XCPT_AC:
5759	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5760	/* no break! */
5761	default:
5762	{
5763	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5764	break;
5765	}
5766	}
5767	}
5768	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5769	{
5770	if (uVector != X86_XCPT_NMI)
5771	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5772	else
5773	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5774	}
5775	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5776	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5777	else
5778	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5779
5780	rc = TRPMResetTrap(pVCpu);
5781	AssertRC(rc);
5782	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5783	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5784	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5785	}
5786
5787
5788	/**
5789	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5790	* VT-x to execute any instruction.
5791	*
5792	* @param pvCpu Pointer to the VMCPU.
5793	*/
5794	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5795	{
5796	Assert(pVCpu->hm.s.Event.fPending);
5797
5798	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5799	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5800	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5801	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5802
5803	/* If a trap was already pending, we did something wrong! */
5804	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5805
5806	TRPMEVENT enmTrapType;
5807	switch (uVectorType)
5808	{
5809	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5810	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5811	enmTrapType = TRPM_HARDWARE_INT;
5812	break;
5813	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5814	enmTrapType = TRPM_SOFTWARE_INT;
5815	break;
5816	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5817	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5818	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5819	enmTrapType = TRPM_TRAP;
5820	break;
5821	default:
5822	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5823	enmTrapType = TRPM_32BIT_HACK;
5824	break;
5825	}
5826
5827	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5828
5829	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5830	AssertRC(rc);
5831
5832	if (fErrorCodeValid)
5833	TRPMSetErrorCode(pVCpu, uErrorCode);
5834	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5835	&& uVector == X86_XCPT_PF)
5836	{
5837	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5838	}
5839	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5840	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5841	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5842	{
5843	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5844	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5845	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5846	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5847	}
5848	pVCpu->hm.s.Event.fPending = false;
5849	}
5850
5851
5852	/**
5853	* Does the necessary state syncing before doing a longjmp to ring-3.
5854	*
5855	* @param pVM Pointer to the VM.
5856	* @param pVCpu Pointer to the VMCPU.
5857	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5858	* out-of-sync. Make sure to update the required fields
5859	* before using them.
5860	* @param rcExit The reason for exiting to ring-3. Can be
5861	* VINF_VMM_UNKNOWN_RING3_CALL.
5862	*
5863	* @remarks No-long-jmp zone!!!
5864	*/
5865	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5866	{
5867	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5868	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5869
5870	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5871	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5872	AssertRC(rc);
5873
5874	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5875	if (CPUMIsGuestFPUStateActive(pVCpu))
5876	{
5877	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5878	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5879	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5880	}
5881
5882	/* Restore debug registers if necessary and resync on next R0 reentry. */
5883	if (CPUMIsGuestDebugStateActive(pVCpu))
5884	{
5885	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5886	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5887	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5888	}
5889	else if (CPUMIsHyperDebugStateActive(pVCpu))
5890	{
5891	CPUMR0LoadHostDebugState(pVM, pVCpu);
5892	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5893	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5894	}
5895
5896	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5897	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
5898	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5899	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5900	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5901	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5902	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5903	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5904	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5905	}
5906
5907
5908	/**
5909	* An action requires us to go back to ring-3. This function does the necessary
5910	* steps before we can safely return to ring-3. This is not the same as longjmps
5911	* to ring-3, this is voluntary.
5912	*
5913	* @param pVM Pointer to the VM.
5914	* @param pVCpu Pointer to the VMCPU.
5915	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5916	* out-of-sync. Make sure to update the required fields
5917	* before using them.
5918	* @param rcExit The reason for exiting to ring-3. Can be
5919	* VINF_VMM_UNKNOWN_RING3_CALL.
5920	*/
5921	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5922	{
5923	Assert(pVM);
5924	Assert(pVCpu);
5925	Assert(pMixedCtx);
5926	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5927
5928	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5929	{
5930	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5931	return;
5932	}
5933	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5934	{
5935	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5936	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5937	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5938	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5939	return;
5940	}
5941
5942	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5943	VMMRZCallRing3Disable(pVCpu);
5944	Log4(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5945
5946	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5947	if (pVCpu->hm.s.Event.fPending)
5948	{
5949	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5950	Assert(!pVCpu->hm.s.Event.fPending);
5951	}
5952
5953	/* Sync. the guest state. */
5954	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5955	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5956
5957	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5958	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5959	\| CPUM_CHANGED_LDTR
5960	\| CPUM_CHANGED_GDTR
5961	\| CPUM_CHANGED_IDTR
5962	\| CPUM_CHANGED_TR
5963	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5964
5965	/* On our way back from ring-3 the following needs to be done. */
5966	/** @todo This can change with preemption hooks. */
5967	if (rcExit == VINF_EM_RAW_INTERRUPT)
5968	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5969	else
5970	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5971
5972	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5973	VMMRZCallRing3Enable(pVCpu);
5974	}
5975
5976
5977	/**
5978	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5979	* longjump to ring-3 and possibly get preempted.
5980	*
5981	* @param pVCpu Pointer to the VMCPU.
5982	* @param enmOperation The operation causing the ring-3 longjump.
5983	* @param pvUser The user argument (pointer to the possibly
5984	* out-of-date guest-CPU context).
5985	*
5986	* @remarks Must never be called with @a enmOperation ==
5987	* VMMCALLRING3_VM_R0_ASSERTION.
5988	*/
5989	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5990	{
5991	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5992	Assert(pVCpu);
5993	Assert(pvUser);
5994	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5995	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5996
5997	VMMRZCallRing3Disable(pVCpu);
5998	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5999	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
6000	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6001	VMMRZCallRing3Enable(pVCpu);
6002	}
6003
6004
6005	/**
6006	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6007	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6008	*
6009	* @returns VBox status code.
6010	* @param pVCpu Pointer to the VMCPU.
6011	*/
6012	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6013	{
6014	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6015	{
6016	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6017	{
6018	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6019	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6020	AssertRC(rc);
6021	}
6022	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6023	}
6024
6025
6026	/**
6027	* Injects any pending events into the guest if the guest is in a state to
6028	* receive them.
6029	*
6030	* @returns VBox status code (informational status codes included).
6031	* @param pVCpu Pointer to the VMCPU.
6032	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6033	* out-of-sync. Make sure to update the required fields
6034	* before using them.
6035	*/
6036	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6037	{
6038	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6039	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6040	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6041	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6042
6043	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6044	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6045	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6046	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6047	Assert(!TRPMHasTrap(pVCpu));
6048
6049	int rc = VINF_SUCCESS;
6050	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6051	{
6052	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6053	bool fInject = true;
6054	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6055	{
6056	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6057	AssertRCReturn(rc, rc);
6058	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6059	if ( fBlockInt
6060	\|\| fBlockSti
6061	\|\| fBlockMovSS)
6062	{
6063	fInject = false;
6064	}
6065	}
6066	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6067	&& ( fBlockMovSS
6068	\|\| fBlockSti))
6069	{
6070	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6071	fInject = false;
6072	}
6073
6074	if (fInject)
6075	{
6076	Log4(("Injecting pending event\n"));
6077	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6078	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6079	AssertRCReturn(rc, rc);
6080	pVCpu->hm.s.Event.fPending = false;
6081	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
6082	}
6083	else
6084	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6085	} /** @todo SMI. SMIs take priority over NMIs. */
6086	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6087	{
6088	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6089	if ( !fBlockMovSS
6090	&& !fBlockSti)
6091	{
6092	Log4(("Injecting NMI\n"));
6093	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6094	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6095	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6096	0 /* GCPtrFaultAddress */, &uIntrState);
6097	AssertRCReturn(rc, rc);
6098	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6099	}
6100	else
6101	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6102	}
6103	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6104	{
6105	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6106	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6107	AssertRCReturn(rc, rc);
6108	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6109	if ( !fBlockInt
6110	&& !fBlockSti
6111	&& !fBlockMovSS)
6112	{
6113	uint8_t u8Interrupt;
6114	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6115	if (RT_SUCCESS(rc))
6116	{
6117	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6118	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6119	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6120	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6121	0 /* GCPtrFaultAddress */, &uIntrState);
6122	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6123	}
6124	else
6125	{
6126	/** @todo Does this actually happen? If not turn it into an assertion. */
6127	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6128	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6129	rc = VINF_SUCCESS;
6130	}
6131	}
6132	else
6133	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6134	}
6135
6136	/*
6137	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6138	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6139	*/
6140	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6141	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6142	int rc2 = VINF_SUCCESS;
6143	if ( fBlockSti
6144	\|\| fBlockMovSS)
6145	{
6146	if (!DBGFIsStepping(pVCpu))
6147	{
6148	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6149	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6150	{
6151	/*
6152	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6153	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6154	*/
6155	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6156	AssertRCReturn(rc, rc);
6157	}
6158	}
6159	else
6160	{
6161	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6162	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6163	uIntrState = 0;
6164	}
6165	}
6166
6167	/*
6168	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6169	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6170	*/
6171	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6172	AssertRC(rc2);
6173
6174	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6175	return rc;
6176	}
6177
6178
6179	/**
6180	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6181	*
6182	* @param pVCpu Pointer to the VMCPU.
6183	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6184	* out-of-sync. Make sure to update the required fields
6185	* before using them.
6186	*/
6187	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6188	{
6189	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6190	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6191	}
6192
6193
6194	/**
6195	* Injects a double-fault (#DF) exception into the VM.
6196	*
6197	* @returns VBox status code (informational status code included).
6198	* @param pVCpu Pointer to the VMCPU.
6199	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6200	* out-of-sync. Make sure to update the required fields
6201	* before using them.
6202	*/
6203	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6204	{
6205	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6206	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6207	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6208	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6209	puIntrState);
6210	}
6211
6212
6213	/**
6214	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6215	*
6216	* @param pVCpu Pointer to the VMCPU.
6217	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6218	* out-of-sync. Make sure to update the required fields
6219	* before using them.
6220	*/
6221	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6222	{
6223	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6224	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6225	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6226	}
6227
6228
6229	/**
6230	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6231	*
6232	* @param pVCpu Pointer to the VMCPU.
6233	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6234	* out-of-sync. Make sure to update the required fields
6235	* before using them.
6236	* @param cbInstr The value of RIP that is to be pushed on the guest
6237	* stack.
6238	*/
6239	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6240	{
6241	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6242	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6243	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6244	}
6245
6246
6247	/**
6248	* Injects a general-protection (#GP) fault into the VM.
6249	*
6250	* @returns VBox status code (informational status code included).
6251	* @param pVCpu Pointer to the VMCPU.
6252	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6253	* out-of-sync. Make sure to update the required fields
6254	* before using them.
6255	* @param u32ErrorCode The error code associated with the #GP.
6256	*/
6257	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6258	uint32_t *puIntrState)
6259	{
6260	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6261	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6262	if (fErrorCodeValid)
6263	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6264	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6265	puIntrState);
6266	}
6267
6268
6269	/**
6270	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6271	*
6272	* @param pVCpu Pointer to the VMCPU.
6273	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6274	* out-of-sync. Make sure to update the required fields
6275	* before using them.
6276	* @param uVector The software interrupt vector number.
6277	* @param cbInstr The value of RIP that is to be pushed on the guest
6278	* stack.
6279	*/
6280	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6281	{
6282	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6283	if ( uVector == X86_XCPT_BP
6284	\|\| uVector == X86_XCPT_OF)
6285	{
6286	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6287	}
6288	else
6289	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6290	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6291	}
6292
6293
6294	/**
6295	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6296	* stack.
6297	*
6298	* @returns VBox status code (information status code included).
6299	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6300	* @param pVM Pointer to the VM.
6301	* @param pMixedCtx Pointer to the guest-CPU context.
6302	* @param uValue The value to push to the guest stack.
6303	*/
6304	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6305	{
6306	/*
6307	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6308	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6309	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6310	*/
6311	if (pMixedCtx->sp == 1)
6312	return VINF_EM_RESET;
6313	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6314	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6315	AssertRCReturn(rc, rc);
6316	return rc;
6317	}
6318
6319
6320	/**
6321	* Injects an event into the guest upon VM-entry by updating the relevant fields
6322	* in the VM-entry area in the VMCS.
6323	*
6324	* @returns VBox status code (informational error codes included).
6325	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6326	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6327	*
6328	* @param pVCpu Pointer to the VMCPU.
6329	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6330	* be out-of-sync. Make sure to update the required
6331	* fields before using them.
6332	* @param u64IntrInfo The VM-entry interruption-information field.
6333	* @param cbInstr The VM-entry instruction length in bytes (for
6334	* software interrupts, exceptions and privileged
6335	* software exceptions).
6336	* @param u32ErrCode The VM-entry exception error code.
6337	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6338	* @param puIntrState Pointer to the current guest interruptibility-state.
6339	* This interruptibility-state will be updated if
6340	* necessary. This cannot not be NULL.
6341	*
6342	* @remarks No-long-jump zone!!!
6343	* @remarks Requires CR0!
6344	*/
6345	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6346	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6347	{
6348	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6349	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6350	Assert(puIntrState);
6351	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6352
6353	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6354	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6355
6356	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6357	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6358	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6359
6360	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6361
6362	/* We require CR0 to check if the guest is in real-mode. */
6363	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6364	AssertRCReturn(rc, rc);
6365
6366	/*
6367	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6368	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6369	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6370	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6371	*/
6372	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6373	{
6374	PVM pVM = pVCpu->CTX_SUFF(pVM);
6375	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6376	{
6377	Assert(PDMVmmDevHeapIsEnabled(pVM));
6378	Assert(pVM->hm.s.vmx.pRealModeTSS);
6379
6380	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6381	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6382	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6383	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6384	AssertRCReturn(rc, rc);
6385	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6386
6387	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6388	const size_t cbIdtEntry = 4;
6389	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6390	{
6391	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6392	if (uVector == X86_XCPT_DF)
6393	return VINF_EM_RESET;
6394	else if (uVector == X86_XCPT_GP)
6395	{
6396	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6397	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6398	}
6399
6400	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6401	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6402	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6403	}
6404
6405	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6406	uint16_t uGuestIp = pMixedCtx->ip;
6407	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6408	{
6409	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6410	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6411	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6412	}
6413	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6414	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6415
6416	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6417	uint16_t offIdtEntry = 0;
6418	RTSEL selIdtEntry = 0;
6419	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6420	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6421	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6422	AssertRCReturn(rc, rc);
6423
6424	/* Construct the stack frame for the interrupt/exception handler. */
6425	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6426	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6427	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6428	AssertRCReturn(rc, rc);
6429
6430	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6431	if (rc == VINF_SUCCESS)
6432	{
6433	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6434	pMixedCtx->rip = offIdtEntry;
6435	pMixedCtx->cs.Sel = selIdtEntry;
6436	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6437	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6438	&& uVector == X86_XCPT_PF)
6439	{
6440	pMixedCtx->cr2 = GCPtrFaultAddress;
6441	}
6442	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6443	\| HM_CHANGED_GUEST_RIP
6444	\| HM_CHANGED_GUEST_RFLAGS
6445	\| HM_CHANGED_GUEST_RSP;
6446
6447	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6448	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6449	{
6450	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6451	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6452	Log4(("Clearing inhibition due to STI.\n"));
6453	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6454	}
6455	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6456	}
6457	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6458	return rc;
6459	}
6460	else
6461	{
6462	/*
6463	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6464	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6465	*/
6466	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6467	}
6468	}
6469
6470	/* Validate. */
6471	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6472	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6473	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6474
6475	/* Inject. */
6476	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6477	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6478	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6479	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6480
6481	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6482	&& uVector == X86_XCPT_PF)
6483	{
6484	pMixedCtx->cr2 = GCPtrFaultAddress;
6485	}
6486
6487	Log4(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr,
6488	pMixedCtx->cr2));
6489
6490	AssertRCReturn(rc, rc);
6491	return rc;
6492	}
6493
6494
6495	/**
6496	* Enters the VT-x session.
6497	*
6498	* @returns VBox status code.
6499	* @param pVM Pointer to the VM.
6500	* @param pVCpu Pointer to the VMCPU.
6501	* @param pCpu Pointer to the CPU info struct.
6502	*/
6503	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6504	{
6505	AssertPtr(pVM);
6506	AssertPtr(pVCpu);
6507	Assert(pVM->hm.s.vmx.fSupported);
6508	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6509	NOREF(pCpu);
6510
6511	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6512
6513	/* Make sure we're in VMX root mode. */
6514	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6515	if (!(u32HostCR4 & X86_CR4_VMXE))
6516	{
6517	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6518	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6519	}
6520
6521	/* Load the active VMCS as the current one. */
6522	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6523	if (RT_FAILURE(rc))
6524	return rc;
6525
6526	/** @todo this will change with preemption hooks where can can VMRESUME as long
6527	* as we're no preempted. */
6528	pVCpu->hm.s.fResumeVM = false;
6529	return VINF_SUCCESS;
6530	}
6531
6532
6533	/**
6534	* Leaves the VT-x session.
6535	*
6536	* @returns VBox status code.
6537	* @param pVM Pointer to the VM.
6538	* @param pVCpu Pointer to the VMCPU.
6539	* @param pCtx Pointer to the guest-CPU context.
6540	*/
6541	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6542	{
6543	AssertPtr(pVCpu);
6544	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6545	NOREF(pVM);
6546	NOREF(pCtx);
6547
6548	/** @todo this will change with preemption hooks where we only VMCLEAR when
6549	* we are actually going to be preempted, not all the time like we
6550	* currently do. */
6551
6552	/* Restore host-state bits that VT-x only restores partially. */
6553	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6554	{
6555	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6556	/** @todo r=ramshankar: This is broken when
6557	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6558	* VMXRestoreHostState() may unconditionally enables interrupts. */
6559	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6560	#else
6561	Assert(ASMIntAreEnabled());
6562	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6563	#endif
6564	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6565	}
6566
6567	/*
6568	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6569	* and mark the VMCS launch-state as "clear".
6570	*/
6571	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6572	return rc;
6573	}
6574
6575
6576	/**
6577	* Saves the host state in the VMCS host-state.
6578	* Sets up the VM-exit MSR-load area.
6579	*
6580	* The CPU state will be loaded from these fields on every successful VM-exit.
6581	*
6582	* @returns VBox status code.
6583	* @param pVM Pointer to the VM.
6584	* @param pVCpu Pointer to the VMCPU.
6585	*
6586	* @remarks No-long-jump zone!!!
6587	*/
6588	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6589	{
6590	AssertPtr(pVM);
6591	AssertPtr(pVCpu);
6592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6593
6594	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6595
6596	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6597	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6598	return VINF_SUCCESS;
6599
6600	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6601	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6602
6603	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6604	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6605
6606	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6607	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6608
6609	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6610	return rc;
6611	}
6612
6613
6614	/**
6615	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6616	* loaded from these fields on every successful VM-entry.
6617	*
6618	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6619	* Sets up the VM-entry controls.
6620	* Sets up the appropriate VMX non-root function to execute guest code based on
6621	* the guest CPU mode.
6622	*
6623	* @returns VBox status code.
6624	* @param pVM Pointer to the VM.
6625	* @param pVCpu Pointer to the VMCPU.
6626	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6627	* out-of-sync. Make sure to update the required fields
6628	* before using them.
6629	*
6630	* @remarks No-long-jump zone!!!
6631	*/
6632	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6633	{
6634	AssertPtr(pVM);
6635	AssertPtr(pVCpu);
6636	AssertPtr(pMixedCtx);
6637	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6638
6639	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6640
6641	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6642
6643	/* Determine real-on-v86 mode. */
6644	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6645	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6646	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6647	{
6648	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6649	}
6650
6651	/*
6652	* Load the guest-state into the VMCS.
6653	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6654	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6655	*/
6656	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6657	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6658
6659	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6660	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6661
6662	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6663	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6664
6665	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6666	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6667
6668	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6669	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6670	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6671
6672	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6673	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6674
6675	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6676	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6677
6678	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6679	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6680
6681	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6682	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6683	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6684
6685	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6686	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6687
6688	/* Clear any unused and reserved bits. */
6689	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_GUEST_CR2
6690	\| HM_CHANGED_VMX_RESERVED1
6691	\| HM_CHANGED_VMX_RESERVED2);
6692
6693	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6694	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6695	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6696
6697	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6698	return rc;
6699	}
6700
6701
6702	/**
6703	* Does the preparations before executing guest code in VT-x.
6704	*
6705	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6706	* recompiler. We must be cautious what we do here regarding committing
6707	* guest-state information into the the VMCS assuming we assuredly execute the
6708	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6709	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6710	* that the recompiler can (and should) use them when it resumes guest
6711	* execution. Otherwise such operations must be done when we can no longer
6712	* exit to ring-3.
6713	*
6714	* @returns VBox status code (informational status codes included).
6715	* @retval VINF_SUCCESS if we can proceed with running the guest.
6716	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6717	* into the guest.
6718	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6719	*
6720	* @param pVM Pointer to the VM.
6721	* @param pVCpu Pointer to the VMCPU.
6722	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6723	* out-of-sync. Make sure to update the required fields
6724	* before using them.
6725	* @param pVmxTransient Pointer to the VMX transient structure.
6726	*
6727	* @remarks Called with preemption disabled.
6728	*/
6729	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6730	{
6731	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6732
6733	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6734	PGMRZDynMapFlushAutoSet(pVCpu);
6735	#endif
6736
6737	/* Check force flag actions that might require us to go back to ring-3. */
6738	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6739	if (rc != VINF_SUCCESS)
6740	return rc;
6741
6742	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6743	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6744	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6745	{
6746	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6747	RTGCPHYS GCPhysApicBase;
6748	GCPhysApicBase = pMixedCtx->msrApicBase;
6749	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6750
6751	/* Unalias any existing mapping. */
6752	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6753	AssertRCReturn(rc, rc);
6754
6755	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6756	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6757	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6758	AssertRCReturn(rc, rc);
6759
6760	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6761	}
6762
6763	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6764	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6765	pVmxTransient->uEFlags = ASMIntDisableFlags();
6766	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6767	{
6768	ASMSetFlags(pVmxTransient->uEFlags);
6769	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6770	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6771	return VINF_EM_RAW_INTERRUPT;
6772	}
6773	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6774	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6775	#endif
6776
6777	/*
6778	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6779	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6780	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6781	*/
6782	/** @todo Rework event evaluation and injection to be completely separate. */
6783	if (TRPMHasTrap(pVCpu))
6784	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6785
6786	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6787	AssertRCReturn(rc, rc);
6788	return rc;
6789	}
6790
6791
6792	/**
6793	* Prepares to run guest code in VT-x and we've committed to doing so. This
6794	* means there is no backing out to ring-3 or anywhere else at this
6795	* point.
6796	*
6797	* @param pVM Pointer to the VM.
6798	* @param pVCpu Pointer to the VMCPU.
6799	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6800	* out-of-sync. Make sure to update the required fields
6801	* before using them.
6802	* @param pVmxTransient Pointer to the VMX transient structure.
6803	*
6804	* @remarks Called with preemption disabled.
6805	* @remarks No-long-jump zone!!!
6806	*/
6807	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6808	{
6809	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6810	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6811
6812	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6813	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6814	pVmxTransient->uEFlags = ASMIntDisableFlags();
6815	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6816	#endif
6817
6818	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6819	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6820	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6821	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6822	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6823	#endif
6824	int rc = VINF_SUCCESS;
6825	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6826	{
6827	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6828	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6829	}
6830	else if (pVCpu->hm.s.fContextUseFlags)
6831	{
6832	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6833	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6834	}
6835	AssertRC(rc);
6836	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6837
6838	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6839	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6840	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6841
6842	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6843	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6844	{
6845	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6846	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6847	}
6848
6849	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6850	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6851	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6852
6853	/*
6854	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6855	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6856	*/
6857	if (pVM->hm.s.fTPRPatchingActive)
6858	{
6859	Assert(!CPUMIsGuestInLongMode(pVCpu));
6860
6861	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6862	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6863	AssertRC(rc);
6864
6865	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6866	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6867	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6868	see hmR0VmxLoadGuestApicState(). */
6869	}
6870
6871	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6872	/*
6873	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6874	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6875	*/
6876	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6877	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6878	{
6879	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6880	uint64_t u64HostTscAux = 0;
6881	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6882	AssertRC(rc2);
6883	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6884	}
6885	#endif
6886
6887	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6888	to start executing. */
6889	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6890	}
6891
6892
6893	/**
6894	* Performs some essential restoration of state after running guest code in
6895	* VT-x.
6896	*
6897	* @param pVM Pointer to the VM.
6898	* @param pVCpu Pointer to the VMCPU.
6899	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6900	* out-of-sync. Make sure to update the required fields
6901	* before using them.
6902	* @param pVmxTransient Pointer to the VMX transient structure.
6903	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6904	*
6905	* @remarks Called with interrupts disabled.
6906	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6907	* unconditionally when it is safe to do so.
6908	*/
6909	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6910	{
6911	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6912	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6913
6914	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6915	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6916	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6917	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6918	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6919
6920	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6921	{
6922	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6923	/* Restore host's TSC_AUX. */
6924	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6925	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6926	#endif
6927	/** @todo Find a way to fix hardcoding a guestimate. */
6928	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6929	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6930	}
6931
6932	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6933	Assert(!(ASMGetFlags() & X86_EFL_IF));
6934	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6935
6936	/* Restore the effects of TPR patching if any. */
6937	if (pVM->hm.s.fTPRPatchingActive)
6938	{
6939	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6940	AssertRC(rc);
6941	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6942	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6943	}
6944
6945	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6946	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6947
6948	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6949	uint32_t uExitReason;
6950	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6951	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6952	AssertRC(rc);
6953	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6954	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6955
6956	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6957	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6958
6959	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6960	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6961	{
6962	Log4(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6963	return;
6964	}
6965
6966	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6967	{
6968	/* Update the guest interruptibility-state from the VMCS. */
6969	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6970	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6971	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6972	AssertRC(rc);
6973	#endif
6974	/*
6975	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6976	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6977	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6978	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6979	*/
6980	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6981	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6982	{
6983	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6984	AssertRC(rc);
6985	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6986	}
6987	}
6988	}
6989
6990
6991	/**
6992	* Runs the guest code using VT-x.
6993	*
6994	* @returns VBox status code.
6995	* @param pVM Pointer to the VM.
6996	* @param pVCpu Pointer to the VMCPU.
6997	* @param pCtx Pointer to the guest-CPU context.
6998	*
6999	* @remarks Called with preemption disabled.
7000	*/
7001	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7002	{
7003	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7004	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7005
7006	VMXTRANSIENT VmxTransient;
7007	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7008	int rc = VERR_INTERNAL_ERROR_5;
7009	uint32_t cLoops = 0;
7010
7011	for (;; cLoops++)
7012	{
7013	Assert(!HMR0SuspendPending());
7014	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7015	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7016	(unsigned)RTMpCpuId(), cLoops));
7017
7018	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7019	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7020	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7021	if (rc != VINF_SUCCESS)
7022	break;
7023
7024	/*
7025	* No longjmps to ring-3 from this point on!!!
7026	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7027	* This also disables flushing of the R0-logger instance (if any).
7028	*/
7029	VMMRZCallRing3Disable(pVCpu);
7030	VMMRZCallRing3RemoveNotification(pVCpu);
7031	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7032
7033	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7034	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7035
7036	/*
7037	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7038	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7039	*/
7040	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7041	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7042	{
7043	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7044	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7045	return rc;
7046	}
7047
7048	/* Handle the VM-exit. */
7049	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7050	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7051	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7052	HMVMX_START_EXIT_DISPATCH_PROF();
7053	#ifdef HMVMX_USE_FUNCTION_TABLE
7054	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7055	#else
7056	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7057	#endif
7058	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7059	if (rc != VINF_SUCCESS)
7060	break;
7061	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7062	{
7063	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7064	rc = VINF_EM_RAW_INTERRUPT;
7065	break;
7066	}
7067	}
7068
7069	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7070	if (rc == VERR_EM_INTERPRETER)
7071	rc = VINF_EM_RAW_EMULATE_INSTR;
7072	else if (rc == VINF_EM_RESET)
7073	rc = VINF_EM_TRIPLE_FAULT;
7074	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7075	return rc;
7076	}
7077
7078
7079	#ifndef HMVMX_USE_FUNCTION_TABLE
7080	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7081	{
7082	int rc;
7083	switch (rcReason)
7084	{
7085	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7086	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7087	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7088	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7089	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7090	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7091	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7092	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7093	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7094	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7095	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7096	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7097	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7098	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7099	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7100	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7101	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7102	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7103	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7104	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7105	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7106	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7107	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7108	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7109	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7110	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7111	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7112	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7113	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7114	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7115	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7116	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7117	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7118
7119	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7120	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7121	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7122	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7123	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7124	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7125	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7126	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7127	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7128
7129	case VMX_EXIT_VMCALL:
7130	case VMX_EXIT_VMCLEAR:
7131	case VMX_EXIT_VMLAUNCH:
7132	case VMX_EXIT_VMPTRLD:
7133	case VMX_EXIT_VMPTRST:
7134	case VMX_EXIT_VMREAD:
7135	case VMX_EXIT_VMRESUME:
7136	case VMX_EXIT_VMWRITE:
7137	case VMX_EXIT_VMXOFF:
7138	case VMX_EXIT_VMXON:
7139	case VMX_EXIT_INVEPT:
7140	case VMX_EXIT_INVVPID:
7141	case VMX_EXIT_VMFUNC:
7142	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7143	break;
7144	default:
7145	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7146	break;
7147	}
7148	return rc;
7149	}
7150	#endif
7151
7152	#ifdef DEBUG
7153	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7154	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7155	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7156	# define VMX_ASSERT_PREEMPT_CPUID() \
7157	do \
7158	{ \
7159	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7160	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7161	} while (0)
7162
7163	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7164	do { \
7165	AssertPtr(pVCpu); \
7166	AssertPtr(pMixedCtx); \
7167	AssertPtr(pVmxTransient); \
7168	Assert(pVmxTransient->fVMEntryFailed == false); \
7169	Assert(ASMIntAreEnabled()); \
7170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7171	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7172	Log4Func(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7173	(unsigned)pVCpu->idCpu)); \
7174	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7175	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7176	VMX_ASSERT_PREEMPT_CPUID(); \
7177	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7178	} while (0)
7179	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7180	do { \
7181	Log4Func(("\n")); \
7182	} while(0)
7183	#else /* Release builds */
7184	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7185	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7186	#endif
7187
7188
7189	/**
7190	* Advances the guest RIP after reading it from the VMCS.
7191	*
7192	* @returns VBox status code.
7193	* @param pVCpu Pointer to the VMCPU.
7194	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7195	* out-of-sync. Make sure to update the required fields
7196	* before using them.
7197	* @param pVmxTransient Pointer to the VMX transient structure.
7198	*
7199	* @remarks No-long-jump zone!!!
7200	*/
7201	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7202	{
7203	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7204	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7205	AssertRCReturn(rc, rc);
7206
7207	pMixedCtx->rip += pVmxTransient->cbInstr;
7208	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7209	return rc;
7210	}
7211
7212
7213	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7214	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7215	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7216	/**
7217	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7218	*/
7219	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7220	{
7221	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7222	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7223	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7224	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7225	Assert(ASMIntAreEnabled());
7226	return VINF_SUCCESS;
7227	#else
7228	return VINF_EM_RAW_INTERRUPT;
7229	#endif
7230	}
7231
7232
7233	/**
7234	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7235	*/
7236	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7237	{
7238	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7239	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7240
7241	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7242	AssertRCReturn(rc, rc);
7243
7244	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7245	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7246	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7247
7248	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7249	{
7250	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7251	return VINF_EM_RAW_INTERRUPT;
7252	}
7253
7254	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7255	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7256	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7257	{
7258	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7259	return VINF_SUCCESS;
7260	}
7261	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7262	{
7263	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7264	return rc;
7265	}
7266
7267	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7268	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7269	switch (uIntrType)
7270	{
7271	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7272	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7273	/* no break */
7274	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7275	{
7276	switch (uVector)
7277	{
7278	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7279	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7280	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7281	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7282	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7283	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7284	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7285	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7286	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7287	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7288	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7289	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7290	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7291	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7292	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7293	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7294	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7295	#endif
7296	default:
7297	{
7298	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7299	AssertRCReturn(rc, rc);
7300
7301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7302	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7303	{
7304	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7305	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7306	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7307	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7308	AssertRCReturn(rc, rc);
7309	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7310	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7311	0 /* GCPtrFaultAddress */);
7312	AssertRCReturn(rc, rc);
7313	}
7314	else
7315	{
7316	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7317	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7318	}
7319	break;
7320	}
7321	}
7322	break;
7323	}
7324
7325	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7326	default:
7327	{
7328	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7329	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7330	break;
7331	}
7332	}
7333	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7334	return rc;
7335	}
7336
7337
7338	/**
7339	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7340	*/
7341	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7342	{
7343	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7344
7345	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7346	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7347	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7348	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7349	AssertRCReturn(rc, rc);
7350
7351	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7352	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7353	return VINF_SUCCESS;
7354	}
7355
7356
7357	/**
7358	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7359	*/
7360	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7361	{
7362	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7363	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7364	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7365	}
7366
7367
7368	/**
7369	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7370	*/
7371	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7372	{
7373	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7374	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7375	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7376	}
7377
7378
7379	/**
7380	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7381	*/
7382	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7383	{
7384	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7386	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7387	}
7388
7389
7390	/**
7391	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7392	*/
7393	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7394	{
7395	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7396	PVM pVM = pVCpu->CTX_SUFF(pVM);
7397	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7398	if (RT_LIKELY(rc == VINF_SUCCESS))
7399	{
7400	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7401	Assert(pVmxTransient->cbInstr == 2);
7402	}
7403	else
7404	{
7405	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7406	rc = VERR_EM_INTERPRETER;
7407	}
7408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7409	return rc;
7410	}
7411
7412
7413	/**
7414	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7415	*/
7416	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7417	{
7418	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7419	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7420	AssertRCReturn(rc, rc);
7421
7422	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7423	return VINF_EM_RAW_EMULATE_INSTR;
7424
7425	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7426	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7427	}
7428
7429
7430	/**
7431	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7432	*/
7433	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7434	{
7435	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7436	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7437	AssertRCReturn(rc, rc);
7438
7439	PVM pVM = pVCpu->CTX_SUFF(pVM);
7440	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7441	if (RT_LIKELY(rc == VINF_SUCCESS))
7442	{
7443	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7444	Assert(pVmxTransient->cbInstr == 2);
7445	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7446	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7447	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7448	}
7449	else
7450	{
7451	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7452	rc = VERR_EM_INTERPRETER;
7453	}
7454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7455	return rc;
7456	}
7457
7458
7459	/**
7460	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7461	*/
7462	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7465	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7466	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7467	AssertRCReturn(rc, rc);
7468
7469	PVM pVM = pVCpu->CTX_SUFF(pVM);
7470	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7471	if (RT_LIKELY(rc == VINF_SUCCESS))
7472	{
7473	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7474	Assert(pVmxTransient->cbInstr == 3);
7475	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7476	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7477	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7478	}
7479	else
7480	{
7481	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7482	rc = VERR_EM_INTERPRETER;
7483	}
7484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7485	return rc;
7486	}
7487
7488
7489	/**
7490	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7491	*/
7492	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7493	{
7494	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7495	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7496	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7497	AssertRCReturn(rc, rc);
7498
7499	PVM pVM = pVCpu->CTX_SUFF(pVM);
7500	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7501	if (RT_LIKELY(rc == VINF_SUCCESS))
7502	{
7503	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7504	Assert(pVmxTransient->cbInstr == 2);
7505	}
7506	else
7507	{
7508	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7509	rc = VERR_EM_INTERPRETER;
7510	}
7511	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7512	return rc;
7513	}
7514
7515
7516	/**
7517	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7518	*/
7519	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7520	{
7521	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7522	PVM pVM = pVCpu->CTX_SUFF(pVM);
7523	Assert(!pVM->hm.s.fNestedPaging);
7524
7525	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7526	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7527	AssertRCReturn(rc, rc);
7528
7529	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7530	rc = VBOXSTRICTRC_VAL(rc2);
7531	if (RT_LIKELY(rc == VINF_SUCCESS))
7532	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7533	else
7534	{
7535	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7536	pVmxTransient->uExitQualification, rc));
7537	}
7538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7539	return rc;
7540	}
7541
7542
7543	/**
7544	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7545	*/
7546	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7547	{
7548	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7549	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7550	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7551	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7552	AssertRCReturn(rc, rc);
7553
7554	PVM pVM = pVCpu->CTX_SUFF(pVM);
7555	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7556	if (RT_LIKELY(rc == VINF_SUCCESS))
7557	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7558	else
7559	{
7560	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7561	rc = VERR_EM_INTERPRETER;
7562	}
7563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7564	return rc;
7565	}
7566
7567
7568	/**
7569	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7570	*/
7571	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7572	{
7573	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7574	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7575	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7576	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7577	AssertRCReturn(rc, rc);
7578
7579	PVM pVM = pVCpu->CTX_SUFF(pVM);
7580	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7581	rc = VBOXSTRICTRC_VAL(rc2);
7582	if (RT_LIKELY( rc == VINF_SUCCESS
7583	\|\| rc == VINF_EM_HALT))
7584	{
7585	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7586	AssertRCReturn(rc3, rc3);
7587
7588	if ( rc == VINF_EM_HALT
7589	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7590	{
7591	rc = VINF_SUCCESS;
7592	}
7593	}
7594	else
7595	{
7596	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7597	rc = VERR_EM_INTERPRETER;
7598	}
7599	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7600	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7602	return rc;
7603	}
7604
7605
7606	/**
7607	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7608	*/
7609	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7610	{
7611	/*
7612	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7613	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7614	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7615	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7616	*/
7617	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7618	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7619	}
7620
7621
7622	/**
7623	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7624	*/
7625	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7626	{
7627	/*
7628	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7629	* root operation. If we get there there is something funny going on.
7630	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7631	*/
7632	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7633	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7634	}
7635
7636
7637	/**
7638	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7639	*/
7640	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7641	{
7642	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7643	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7644	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7645	}
7646
7647
7648	/**
7649	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7650	*/
7651	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7652	{
7653	/*
7654	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7655	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7656	* See Intel spec. 25.3 "Other Causes of VM-exits".
7657	*/
7658	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7659	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7660	}
7661
7662
7663	/**
7664	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7665	* VM-exit.
7666	*/
7667	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7668	{
7669	/*
7670	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7671	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7672	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7673	*/
7674	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7675	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7676	}
7677
7678
7679	/**
7680	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7681	* VM-exit.
7682	*/
7683	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7684	{
7685	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7686	return VINF_EM_RESET;
7687	}
7688
7689
7690	/**
7691	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7692	*/
7693	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7694	{
7695	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7696	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7697	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7698	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7699	AssertRCReturn(rc, rc);
7700
7701	pMixedCtx->rip++;
7702	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7703	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7704	rc = VINF_SUCCESS;
7705	else
7706	rc = VINF_EM_HALT;
7707
7708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7709	return rc;
7710	}
7711
7712
7713	/**
7714	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7715	*/
7716	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7717	{
7718	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7719	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7720	return VINF_SUCCESS;
7721	}
7722
7723
7724	/**
7725	* VM-exit handler for expiry of the VMX preemption timer.
7726	*/
7727	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7728	{
7729	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7730
7731	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7732	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7733
7734	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7735	PVM pVM = pVCpu->CTX_SUFF(pVM);
7736	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7738	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7739	}
7740
7741
7742	/**
7743	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7744	*/
7745	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7746	{
7747	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7748
7749	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7750	/** @todo check if XSETBV is supported by the recompiler. */
7751	return VERR_EM_INTERPRETER;
7752	}
7753
7754
7755	/**
7756	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7757	*/
7758	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7759	{
7760	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7761
7762	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7763	/** @todo implement EMInterpretInvpcid() */
7764	return VERR_EM_INTERPRETER;
7765	}
7766
7767
7768	/**
7769	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7770	* Error VM-exit.
7771	*/
7772	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7773	{
7774	uint32_t uIntrState;
7775	HMVMXHCUINTREG uHCReg;
7776	uint64_t u64Val;
7777	uint32_t u32Val;
7778
7779	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7780	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7781	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7782	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7783	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7784	AssertRCReturn(rc, rc);
7785
7786	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7787	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7788	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7789	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7790
7791	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7792	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7793	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7794	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7795	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7796	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7797	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7798	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7799	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7800	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7801	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7802	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7803
7804	PVM pVM = pVCpu->CTX_SUFF(pVM);
7805	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7806
7807	return VERR_VMX_INVALID_GUEST_STATE;
7808	}
7809
7810
7811	/**
7812	* VM-exit handler for VM-entry failure due to an MSR-load
7813	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7814	*/
7815	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7816	{
7817	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7818	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7819	}
7820
7821
7822	/**
7823	* VM-exit handler for VM-entry failure due to a machine-check event
7824	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7825	*/
7826	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7827	{
7828	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7829	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7830	}
7831
7832
7833	/**
7834	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7835	* theory.
7836	*/
7837	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7838	{
7839	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7840	return VERR_VMX_UNDEFINED_EXIT_CODE;
7841	}
7842
7843
7844	/**
7845	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7846	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7847	* Conditional VM-exit.
7848	*/
7849	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7850	{
7851	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7852
7853	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7854	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7855	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7856	return VERR_EM_INTERPRETER;
7857	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7858	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7859	}
7860
7861
7862	/**
7863	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7864	*/
7865	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7866	{
7867	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7868
7869	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7871	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7872	return VERR_EM_INTERPRETER;
7873	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7874	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7875	}
7876
7877
7878	/**
7879	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7880	*/
7881	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7882	{
7883	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7884
7885	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7886	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7887	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7888	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7889	AssertRCReturn(rc, rc);
7890
7891	PVM pVM = pVCpu->CTX_SUFF(pVM);
7892	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7893	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7894	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7896
7897	if (RT_LIKELY(rc == VINF_SUCCESS))
7898	{
7899	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7900	Assert(pVmxTransient->cbInstr == 2);
7901	}
7902	return rc;
7903	}
7904
7905
7906	/**
7907	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7908	*/
7909	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7910	{
7911	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7912	PVM pVM = pVCpu->CTX_SUFF(pVM);
7913	int rc = VINF_SUCCESS;
7914
7915	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7916	if ( pVM->hm.s.fTPRPatchingActive
7917	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7918	{
7919	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7920	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7921	{
7922	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7923	AssertRC(rc);
7924	}
7925
7926	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7927	Assert(pVmxTransient->cbInstr == 2);
7928	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7929	return VINF_SUCCESS;
7930	}
7931
7932	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7933	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7934	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7935	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7936	AssertRCReturn(rc, rc);
7937	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
7938
7939	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7940	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7942
7943	if (RT_LIKELY(rc == VINF_SUCCESS))
7944	{
7945	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7946
7947	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7948	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7949	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7950	{
7951	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7952	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7953	}
7954	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7955	{
7956	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7957	AssertRCReturn(rc, rc);
7958	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7959	}
7960	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7961	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7962
7963	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7964	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7965	{
7966	switch (pMixedCtx->ecx)
7967	{
7968	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7969	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7970	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7971	case MSR_K8_FS_BASE: /* no break */
7972	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7973	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7974	}
7975	}
7976	#ifdef VBOX_STRICT
7977	else
7978	{
7979	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7980	switch (pMixedCtx->ecx)
7981	{
7982	case MSR_IA32_SYSENTER_CS:
7983	case MSR_IA32_SYSENTER_EIP:
7984	case MSR_IA32_SYSENTER_ESP:
7985	case MSR_K8_FS_BASE:
7986	case MSR_K8_GS_BASE:
7987	{
7988	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7989	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7990	}
7991
7992	case MSR_K8_LSTAR:
7993	case MSR_K6_STAR:
7994	case MSR_K8_SF_MASK:
7995	case MSR_K8_TSC_AUX:
7996	case MSR_K8_KERNEL_GS_BASE:
7997	{
7998	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7999	pMixedCtx->ecx));
8000	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8001	}
8002	}
8003	}
8004	#endif /* VBOX_STRICT */
8005	}
8006	return rc;
8007	}
8008
8009
8010	/**
8011	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8012	*/
8013	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8014	{
8015	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8016
8017	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8018	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8019	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8020	return VERR_EM_INTERPRETER;
8021	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8022	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8023	}
8024
8025
8026	/**
8027	* VM-exit handler for when the TPR value is lowered below the specified
8028	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8029	*/
8030	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8031	{
8032	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8033	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8034
8035	/*
8036	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8037	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8038	* resume guest execution.
8039	*/
8040	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8042	return VINF_SUCCESS;
8043	}
8044
8045
8046	/**
8047	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8048	* VM-exit.
8049	*
8050	* @retval VINF_SUCCESS when guest execution can continue.
8051	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8052	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8053	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8054	* recompiler.
8055	*/
8056	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8057	{
8058	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8059	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8060	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8061	AssertRCReturn(rc, rc);
8062
8063	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8064	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8065	PVM pVM = pVCpu->CTX_SUFF(pVM);
8066	switch (uAccessType)
8067	{
8068	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8069	{
8070	#if 0
8071	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8072	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8073	#else
8074	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8075	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8076	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8077	#endif
8078	AssertRCReturn(rc, rc);
8079
8080	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8081	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8082	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8083	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8084
8085	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8086	{
8087	case 0: /* CR0 */
8088	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8089	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8090	break;
8091	case 2: /* C2 **/
8092	/* Nothing to do here, CR2 it's not part of the VMCS. */
8093	break;
8094	case 3: /* CR3 */
8095	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8096	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8097	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8098	break;
8099	case 4: /* CR4 */
8100	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8101	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8102	break;
8103	case 8: /* CR8 */
8104	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8105	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8106	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8107	break;
8108	default:
8109	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8110	break;
8111	}
8112
8113	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8114	break;
8115	}
8116
8117	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8118	{
8119	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8120	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8121	AssertRCReturn(rc, rc);
8122	Assert( !pVM->hm.s.fNestedPaging
8123	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8124	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8125
8126	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8127	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8128	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8129
8130	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8131	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8132	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8133	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8135	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8136	break;
8137	}
8138
8139	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8140	{
8141	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8142	AssertRCReturn(rc, rc);
8143	rc = EMInterpretCLTS(pVM, pVCpu);
8144	AssertRCReturn(rc, rc);
8145	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8147	Log4(("CRX CLTS write rc=%d\n", rc));
8148	break;
8149	}
8150
8151	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8152	{
8153	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8154	AssertRCReturn(rc, rc);
8155	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8156	if (RT_LIKELY(rc == VINF_SUCCESS))
8157	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8158	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8159	Log4(("CRX LMSW write rc=%d\n", rc));
8160	break;
8161	}
8162
8163	default:
8164	{
8165	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8166	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8167	}
8168	}
8169
8170	/* Validate possible error codes. */
8171	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8172	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8173	if (RT_SUCCESS(rc))
8174	{
8175	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8176	AssertRCReturn(rc2, rc2);
8177	}
8178
8179	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8180	return rc;
8181	}
8182
8183
8184	/**
8185	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8186	* VM-exit.
8187	*/
8188	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8189	{
8190	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8191	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8192
8193	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8194	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8195	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8196	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8197	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8198	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8199	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8200	AssertRCReturn(rc, rc);
8201
8202	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8203
8204	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8205	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8206	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8207	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8208	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8209	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8210	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8211
8212	/* I/O operation lookup arrays. */
8213	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8214	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8215
8216	const uint32_t cbSize = s_aIOSize[uIOWidth];
8217	const uint32_t cbInstr = pVmxTransient->cbInstr;
8218	PVM pVM = pVCpu->CTX_SUFF(pVM);
8219	if (fIOString)
8220	{
8221	/* INS/OUTS - I/O String instruction. */
8222	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8223	/** @todo for now manually disassemble later optimize by getting the fields from
8224	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8225	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8226	* segment prefix info. */
8227	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8228	if (RT_SUCCESS(rc))
8229	{
8230	if (fIOWrite)
8231	{
8232	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8233	(DISCPUMODE)pDis->uAddrMode, cbSize);
8234	rc = VBOXSTRICTRC_VAL(rc2);
8235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8236	}
8237	else
8238	{
8239	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8240	(DISCPUMODE)pDis->uAddrMode, cbSize);
8241	rc = VBOXSTRICTRC_VAL(rc2);
8242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8243	}
8244	}
8245	else
8246	{
8247	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8248	rc = VINF_EM_RAW_EMULATE_INSTR;
8249	}
8250	}
8251	else
8252	{
8253	/* IN/OUT - I/O instruction. */
8254	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8255	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8256	if (fIOWrite)
8257	{
8258	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8259	rc = VBOXSTRICTRC_VAL(rc2);
8260	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8261	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8263	}
8264	else
8265	{
8266	uint32_t u32Result = 0;
8267	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8268	rc = VBOXSTRICTRC_VAL(rc2);
8269	if (IOM_SUCCESS(rc))
8270	{
8271	/* Save result of I/O IN instr. in AL/AX/EAX. */
8272	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8273	}
8274	else if (rc == VINF_IOM_R3_IOPORT_READ)
8275	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8276	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8277	}
8278	}
8279
8280	if (IOM_SUCCESS(rc))
8281	{
8282	pMixedCtx->rip += cbInstr;
8283	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8284	if (RT_LIKELY(rc == VINF_SUCCESS))
8285	{
8286	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8287	AssertRCReturn(rc, rc);
8288
8289	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8290	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8291	{
8292	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8293	for (unsigned i = 0; i < 4; i++)
8294	{
8295	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8296	if ( ( uIOPort >= pMixedCtx->dr[i]
8297	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8298	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8299	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8300	{
8301	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8302	uint64_t uDR6 = ASMGetDR6();
8303
8304	/* Clear all breakpoint status flags and set the one we just hit. */
8305	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8306	uDR6 \|= (uint64_t)RT_BIT(i);
8307
8308	/*
8309	* Note: AMD64 Architecture Programmer's Manual 13.1:
8310	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8311	* be cleared by software after the contents have been read.
8312	*/
8313	ASMSetDR6(uDR6);
8314
8315	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8316	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8317
8318	/* Paranoia. */
8319	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* Upper 32 bits MBZ. */
8320	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8321	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8322
8323	/* Resync DR7 */
8324	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8325	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8326
8327	/* Set #DB to be injected into the VM and continue guest execution. */
8328	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8329	break;
8330	}
8331	}
8332	}
8333	}
8334	}
8335
8336	#ifdef DEBUG
8337	if (rc == VINF_IOM_R3_IOPORT_READ)
8338	Assert(!fIOWrite);
8339	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8340	Assert(fIOWrite);
8341	else
8342	{
8343	AssertMsg( RT_FAILURE(rc)
8344	\|\| rc == VINF_SUCCESS
8345	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8346	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8347	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8348	}
8349	#endif
8350
8351	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8352	return rc;
8353	}
8354
8355
8356	/**
8357	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8358	* VM-exit.
8359	*/
8360	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8361	{
8362	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8363
8364	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8365	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8366	AssertRCReturn(rc, rc);
8367	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8368	{
8369	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8370	AssertRCReturn(rc, rc);
8371	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8372	{
8373	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8374
8375	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8376	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8377	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8378	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8379	{
8380	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8381	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8382
8383	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8384	Assert(!pVCpu->hm.s.Event.fPending);
8385	pVCpu->hm.s.Event.fPending = true;
8386	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8387	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8388	AssertRCReturn(rc, rc);
8389	if (fErrorCodeValid)
8390	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8391	else
8392	pVCpu->hm.s.Event.u32ErrCode = 0;
8393	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8394	&& uVector == X86_XCPT_PF)
8395	{
8396	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8397	}
8398
8399	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8400	}
8401	}
8402	}
8403
8404	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8405	* emulation. */
8406	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8407	return VERR_EM_INTERPRETER;
8408	}
8409
8410
8411	/**
8412	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8413	*/
8414	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8415	{
8416	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8417	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8418	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8419	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8420	AssertRCReturn(rc, rc);
8421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8422	return VINF_EM_DBG_STOP;
8423	}
8424
8425
8426	/**
8427	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8428	*/
8429	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8430	{
8431	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8432
8433	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8434	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8435	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8436	return VINF_SUCCESS;
8437	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8438	return rc;
8439
8440	#if 0
8441	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8442	* just sync the whole thing. */
8443	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8444	#else
8445	/* Aggressive state sync. for now. */
8446	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8447	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8448	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8449	#endif
8450	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8451	AssertRCReturn(rc, rc);
8452
8453	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8454	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8455	switch (uAccessType)
8456	{
8457	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8458	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8459	{
8460	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8461	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8462	{
8463	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8464	}
8465
8466	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8467	GCPhys &= PAGE_BASE_GC_MASK;
8468	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8469	PVM pVM = pVCpu->CTX_SUFF(pVM);
8470	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8471	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8472
8473	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8474	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8475	CPUMCTX2CORE(pMixedCtx), GCPhys);
8476	rc = VBOXSTRICTRC_VAL(rc2);
8477	Log4(("ApicAccess rc=%d\n", rc));
8478	if ( rc == VINF_SUCCESS
8479	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8480	\|\| rc == VERR_PAGE_NOT_PRESENT)
8481	{
8482	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8483	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8484	rc = VINF_SUCCESS;
8485	}
8486	break;
8487	}
8488
8489	default:
8490	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8491	rc = VINF_EM_RAW_EMULATE_INSTR;
8492	break;
8493	}
8494
8495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8496	return rc;
8497	}
8498
8499
8500	/**
8501	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8502	* VM-exit.
8503	*/
8504	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8505	{
8506	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8507
8508	/* We should -not- get this VM-exit if the guest is debugging. */
8509	if (CPUMIsGuestDebugStateActive(pVCpu))
8510	{
8511	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8512	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8513	}
8514
8515	int rc = VERR_INTERNAL_ERROR_5;
8516	if ( !DBGFIsStepping(pVCpu)
8517	&& !CPUMIsHyperDebugStateActive(pVCpu))
8518	{
8519	/* Don't intercept MOV DRx. */
8520	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8521	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8522	AssertRCReturn(rc, rc);
8523
8524	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8525	PVM pVM = pVCpu->CTX_SUFF(pVM);
8526	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8527	AssertRC(rc);
8528	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8529
8530	#ifdef VBOX_WITH_STATISTICS
8531	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8532	AssertRCReturn(rc, rc);
8533	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8535	else
8536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8537	#endif
8538	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8539	return VINF_SUCCESS;
8540	}
8541
8542	/*
8543	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8544	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8545	*/
8546	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8547	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8548	AssertRCReturn(rc, rc);
8549
8550	PVM pVM = pVCpu->CTX_SUFF(pVM);
8551	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8552	{
8553	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8554	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8555	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8556	if (RT_SUCCESS(rc))
8557	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8558	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8559	}
8560	else
8561	{
8562	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8563	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8564	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8566	}
8567
8568	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8569	if (RT_SUCCESS(rc))
8570	{
8571	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8572	AssertRCReturn(rc2, rc2);
8573	}
8574	return rc;
8575	}
8576
8577
8578	/**
8579	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8580	* Conditional VM-exit.
8581	*/
8582	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8583	{
8584	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8585	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8586
8587	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8588	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8589	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8590	return VINF_SUCCESS;
8591	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8592	return rc;
8593
8594	RTGCPHYS GCPhys = 0;
8595	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8596
8597	#if 0
8598	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8599	#else
8600	/* Aggressive state sync. for now. */
8601	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8602	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8603	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8604	#endif
8605	AssertRCReturn(rc, rc);
8606
8607	/*
8608	* If we succeed, resume guest execution.
8609	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8610	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8611	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8612	* weird case. See @bugref{6043}.
8613	*/
8614	PVM pVM = pVCpu->CTX_SUFF(pVM);
8615	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8616	rc = VBOXSTRICTRC_VAL(rc2);
8617	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8618	if ( rc == VINF_SUCCESS
8619	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8620	\|\| rc == VERR_PAGE_NOT_PRESENT)
8621	{
8622	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8623	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8624	return VINF_SUCCESS;
8625	}
8626	return rc;
8627	}
8628
8629
8630	/**
8631	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8632	* VM-exit.
8633	*/
8634	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8635	{
8636	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8637	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8638
8639	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8640	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8641	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8642	return VINF_SUCCESS;
8643	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8644	return rc;
8645
8646	RTGCPHYS GCPhys = 0;
8647	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8648	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8649	#if 0
8650	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8651	#else
8652	/* Aggressive state sync. for now. */
8653	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8654	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8655	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8656	#endif
8657	AssertRCReturn(rc, rc);
8658
8659	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8660	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8661
8662	RTGCUINT uErrorCode = 0;
8663	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8664	uErrorCode \|= X86_TRAP_PF_ID;
8665	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8666	uErrorCode \|= X86_TRAP_PF_RW;
8667	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8668	uErrorCode \|= X86_TRAP_PF_P;
8669
8670	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8671
8672	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8673	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8674
8675	/* Handle the pagefault trap for the nested shadow table. */
8676	PVM pVM = pVCpu->CTX_SUFF(pVM);
8677	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8678	TRPMResetTrap(pVCpu);
8679
8680	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8681	if ( rc == VINF_SUCCESS
8682	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8683	\|\| rc == VERR_PAGE_NOT_PRESENT)
8684	{
8685	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8687	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8688	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8689	return VINF_SUCCESS;
8690	}
8691
8692	Log4(("EPT return to ring-3 rc=%d\n"));
8693	return rc;
8694	}
8695
8696
8697	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8698	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8699	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8700	/**
8701	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8702	*/
8703	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8704	{
8705	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8707
8708	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8709	AssertRCReturn(rc, rc);
8710
8711	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8712	{
8713	/* Old-style FPU error reporting needs some extra work. */
8714	/** @todo don't fall back to the recompiler, but do it manually. */
8715	return VERR_EM_INTERPRETER;
8716	}
8717	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8718	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8719	return rc;
8720	}
8721
8722
8723	/**
8724	* VM-exit exception handler for #BP (Breakpoint exception).
8725	*/
8726	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8727	{
8728	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8730
8731	/** @todo Try optimize this by not saving the entire guest state unless
8732	* really needed. */
8733	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8734	AssertRCReturn(rc, rc);
8735
8736	PVM pVM = pVCpu->CTX_SUFF(pVM);
8737	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8738	if (rc == VINF_EM_RAW_GUEST_TRAP)
8739	{
8740	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8741	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8742	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8743	AssertRCReturn(rc, rc);
8744
8745	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8746	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8747	}
8748
8749	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8750	return rc;
8751	}
8752
8753
8754	/**
8755	* VM-exit exception handler for #DB (Debug exception).
8756	*/
8757	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8758	{
8759	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8760	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8761
8762	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8763	AssertRCReturn(rc, rc);
8764
8765	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8766	uint64_t uDR6 = X86_DR6_INIT_VAL;
8767	uDR6 \|= (pVmxTransient->uExitQualification
8768	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8769	PVM pVM = pVCpu->CTX_SUFF(pVM);
8770	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8771	if (rc == VINF_EM_RAW_GUEST_TRAP)
8772	{
8773	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8774	pMixedCtx->dr[6] = uDR6;
8775
8776	if (CPUMIsGuestDebugStateActive(pVCpu))
8777	ASMSetDR6(pMixedCtx->dr[6]);
8778
8779	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8780
8781	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8782	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8783
8784	/* Paranoia. */
8785	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* upper 32 bits MBZ. */
8786	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8787	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8788
8789	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8790	AssertRCReturn(rc,rc);
8791
8792	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8793	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8794	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8795	AssertRCReturn(rc2, rc2);
8796	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8797	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8798	rc = VINF_SUCCESS;
8799	}
8800
8801	return rc;
8802	}
8803
8804
8805	/**
8806	* VM-exit exception handler for #NM (Device-not-available exception: floating
8807	* point exception).
8808	*/
8809	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8810	{
8811	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8812
8813	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8814	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8815	#endif
8816
8817	/* We require CR0 and EFER. EFER is always up-to-date. */
8818	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8819	AssertRCReturn(rc, rc);
8820
8821	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8822	PVM pVM = pVCpu->CTX_SUFF(pVM);
8823	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8824	if (rc == VINF_SUCCESS)
8825	{
8826	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8827	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8828	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8829	return VINF_SUCCESS;
8830	}
8831
8832	/* Forward #NM to the guest. */
8833	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8834	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8835	AssertRCReturn(rc, rc);
8836	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8837	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8838	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8839	return rc;
8840	}
8841
8842
8843	/**
8844	* VM-exit exception handler for #GP (General-protection exception).
8845	*
8846	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8847	*/
8848	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8849	{
8850	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8852
8853	int rc = VERR_INTERNAL_ERROR_5;
8854	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8855	{
8856	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8857	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8858	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8859	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8860	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8861	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8862	AssertRCReturn(rc, rc);
8863	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8864	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8865	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8866	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8867	return rc;
8868	#else
8869	/* We don't intercept #GP. */
8870	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8871	return VERR_VMX_UNEXPECTED_EXCEPTION;
8872	#endif
8873	}
8874
8875	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8876	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8877
8878	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8879	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8880	AssertRCReturn(rc, rc);
8881
8882	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8883	uint32_t cbOp = 0;
8884	PVM pVM = pVCpu->CTX_SUFF(pVM);
8885	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8886	if (RT_SUCCESS(rc))
8887	{
8888	rc = VINF_SUCCESS;
8889	Assert(cbOp == pDis->cbInstr);
8890	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8891	switch (pDis->pCurInstr->uOpcode)
8892	{
8893	case OP_CLI:
8894	{
8895	pMixedCtx->eflags.Bits.u1IF = 0;
8896	pMixedCtx->rip += pDis->cbInstr;
8897	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8899	break;
8900	}
8901
8902	case OP_STI:
8903	{
8904	pMixedCtx->eflags.Bits.u1IF = 1;
8905	pMixedCtx->rip += pDis->cbInstr;
8906	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8907	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8908	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8910	break;
8911	}
8912
8913	case OP_HLT:
8914	{
8915	rc = VINF_EM_HALT;
8916	pMixedCtx->rip += pDis->cbInstr;
8917	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8918	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8919	break;
8920	}
8921
8922	case OP_POPF:
8923	{
8924	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8925	uint32_t cbParm = 0;
8926	uint32_t uMask = 0;
8927	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8928	{
8929	cbParm = 4;
8930	uMask = 0xffffffff;
8931	}
8932	else
8933	{
8934	cbParm = 2;
8935	uMask = 0xffff;
8936	}
8937
8938	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8939	RTGCPTR GCPtrStack = 0;
8940	X86EFLAGS uEflags;
8941	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8942	&GCPtrStack);
8943	if (RT_SUCCESS(rc))
8944	{
8945	Assert(sizeof(uEflags.u32) >= cbParm);
8946	uEflags.u32 = 0;
8947	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8948	}
8949	if (RT_FAILURE(rc))
8950	{
8951	rc = VERR_EM_INTERPRETER;
8952	break;
8953	}
8954	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8955	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8956	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8957	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8958	pMixedCtx->eflags.Bits.u1RF = 0;
8959	pMixedCtx->esp += cbParm;
8960	pMixedCtx->esp &= uMask;
8961	pMixedCtx->rip += pDis->cbInstr;
8962	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8964	break;
8965	}
8966
8967	case OP_PUSHF:
8968	{
8969	uint32_t cbParm = 0;
8970	uint32_t uMask = 0;
8971	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8972	{
8973	cbParm = 4;
8974	uMask = 0xffffffff;
8975	}
8976	else
8977	{
8978	cbParm = 2;
8979	uMask = 0xffff;
8980	}
8981
8982	/* Get the stack pointer & push the contents of eflags onto the stack. */
8983	RTGCPTR GCPtrStack = 0;
8984	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8985	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8986	if (RT_FAILURE(rc))
8987	{
8988	rc = VERR_EM_INTERPRETER;
8989	break;
8990	}
8991	X86EFLAGS uEflags;
8992	uEflags = pMixedCtx->eflags;
8993	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8994	uEflags.Bits.u1RF = 0;
8995	uEflags.Bits.u1VM = 0;
8996
8997	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8998	if (RT_FAILURE(rc))
8999	{
9000	rc = VERR_EM_INTERPRETER;
9001	break;
9002	}
9003	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9004	pMixedCtx->esp -= cbParm;
9005	pMixedCtx->esp &= uMask;
9006	pMixedCtx->rip += pDis->cbInstr;
9007	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9008	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9009	break;
9010	}
9011
9012	case OP_IRET:
9013	{
9014	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9015	* instruction reference. */
9016	RTGCPTR GCPtrStack = 0;
9017	uint32_t uMask = 0xffff;
9018	uint16_t aIretFrame[3];
9019	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9020	{
9021	rc = VERR_EM_INTERPRETER;
9022	break;
9023	}
9024	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9025	&GCPtrStack);
9026	if (RT_SUCCESS(rc))
9027	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9028	if (RT_FAILURE(rc))
9029	{
9030	rc = VERR_EM_INTERPRETER;
9031	break;
9032	}
9033	pMixedCtx->eip = 0;
9034	pMixedCtx->ip = aIretFrame[0];
9035	pMixedCtx->cs.Sel = aIretFrame[1];
9036	pMixedCtx->cs.ValidSel = aIretFrame[1];
9037	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9038	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9039	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9040	pMixedCtx->sp += sizeof(aIretFrame);
9041	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9042	\| HM_CHANGED_GUEST_RFLAGS;
9043	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9044	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9045	break;
9046	}
9047
9048	case OP_INT:
9049	{
9050	uint16_t uVector = pDis->Param1.uValue & 0xff;
9051	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9053	break;
9054	}
9055
9056	case OP_INTO:
9057	{
9058	if (pMixedCtx->eflags.Bits.u1OF)
9059	{
9060	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9061	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9062	}
9063	break;
9064	}
9065
9066	default:
9067	{
9068	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9069	EMCODETYPE_SUPERVISOR);
9070	rc = VBOXSTRICTRC_VAL(rc2);
9071	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9072	Log4(("#GP rc=%Rrc\n", rc));
9073	break;
9074	}
9075	}
9076	}
9077	else
9078	rc = VERR_EM_INTERPRETER;
9079
9080	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9081	("#GP Unexpected rc=%Rrc\n", rc));
9082	return rc;
9083	}
9084
9085
9086	/**
9087	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9088	* the exception reported in the VMX transient structure back into the VM.
9089	*
9090	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9091	* up-to-date.
9092	*/
9093	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9094	{
9095	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9096
9097	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9098	hmR0VmxCheckExitDueToEventDelivery(). */
9099	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9100	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9101	AssertRCReturn(rc, rc);
9102	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9103	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9104	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9105	return VINF_SUCCESS;
9106	}
9107
9108
9109	/**
9110	* VM-exit exception handler for #PF (Page-fault exception).
9111	*/
9112	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9113	{
9114	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9115	PVM pVM = pVCpu->CTX_SUFF(pVM);
9116	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9117	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9118	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9119	AssertRCReturn(rc, rc);
9120
9121	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9122	if (pVM->hm.s.fNestedPaging)
9123	{
9124	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9125	{
9126	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9127	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9128	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9129	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9131	}
9132	else
9133	{
9134	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9135	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
9136	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9137	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9138	}
9139	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9140	return rc;
9141	}
9142	#else
9143	Assert(!pVM->hm.s.fNestedPaging);
9144	#endif
9145
9146	#ifdef VBOX_HM_WITH_GUEST_PATCHING
9147	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9148	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9149	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9150	AssertRCReturn(rc, rc);
9151	/* Shortcut for APIC TPR access, only for 32-bit guests. */
9152	if ( pVM->hm.s.fTRPPatchingAllowed
9153	&& pVM->hm.s.pGuestPatchMem
9154	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
9155	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
9156	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
9157	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
9158	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
9159	{
9160	RTGCPHYS GCPhys;
9161	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
9162	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9163	if ( rc == VINF_SUCCESS
9164	&& GCPhys == GCPhysApicBase)
9165	{
9166	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9167	AssertRCReturn(rc, rc);
9168
9169	/* Only attempt to patch the instruction once. */
9170	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9171	if (!pPatch)
9172	return VINF_EM_HM_PATCH_TPR_INSTR;
9173	}
9174	}
9175	#endif
9176
9177	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9178	AssertRCReturn(rc, rc);
9179
9180	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9181	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9182
9183	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9184	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9185	(RTGCPTR)pVmxTransient->uExitQualification);
9186
9187	Log4(("#PF: rc=%Rrc\n", rc));
9188	if (rc == VINF_SUCCESS)
9189	{
9190	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9191	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9192	* memory? We don't update the whole state here... */
9193	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9194	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9195	TRPMResetTrap(pVCpu);
9196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9197	return rc;
9198	}
9199	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9200	{
9201	if (!pVmxTransient->fVectoringPF)
9202	{
9203	/* It's a guest page fault and needs to be reflected to the guest. */
9204	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9205	TRPMResetTrap(pVCpu);
9206	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9207	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9208	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9209	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9210	}
9211	else
9212	{
9213	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9214	TRPMResetTrap(pVCpu);
9215	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9216	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9217	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9218	}
9219
9220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9221	return VINF_SUCCESS;
9222	}
9223
9224	TRPMResetTrap(pVCpu);
9225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9226	return rc;
9227	}
9228

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46443

Download in other formats: