HMVMXR0.cpp@ 46664

Last change on this file since 46664 was 46655, checked in by vboxsync, 12 years ago

VMM/HMVMXR0: Reflect any exception to the guest if event delivery of that exception caused an EPT violation/misconfig. of APIC-access VM-exit.

In this case the VM-exit interruption-information field won't be valid because the resulting VM-exit is -not- an exception.

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 384.0 KB

Line
1	/* $Id: HMVMXR0.cpp 46655 2013-06-19 12:46:36Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/** @name VMX transient.
163	*
164	* A state structure for holding miscellaneous information across
165	* VMX non-root operation and restored after the transition.
166	*
167	* @{ */
168	typedef struct VMXTRANSIENT
169	{
170	/** The host's rflags/eflags. */
171	RTCCUINTREG uEFlags;
172	#if HC_ARCH_BITS == 32
173	uint32_t u32Alignment0;
174	#endif
175	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
176	uint64_t u64LStarMsr;
177	/** The guest's TPR value used for TPR shadowing. */
178	uint8_t u8GuestTpr;
179	/** Alignment. */
180	uint8_t abAlignment0[6];
181
182	/** The basic VM-exit reason. */
183	uint16_t uExitReason;
184	/** Alignment. */
185	uint16_t u16Alignment0;
186	/** The VM-exit interruption error code. */
187	uint32_t uExitIntrErrorCode;
188	/** The VM-exit exit qualification. */
189	uint64_t uExitQualification;
190
191	/** The VM-exit interruption-information field. */
192	uint32_t uExitIntrInfo;
193	/** The VM-exit instruction-length field. */
194	uint32_t cbInstr;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Alignment. */
198	uint8_t abAlignment1[5];
199
200	/** The VM-entry interruption-information field. */
201	uint32_t uEntryIntrInfo;
202	/** The VM-entry exception error code field. */
203	uint32_t uEntryXcptErrorCode;
204	/** The VM-entry instruction length field. */
205	uint32_t cbEntryInstr;
206
207	/** IDT-vectoring information field. */
208	uint32_t uIdtVectoringInfo;
209	/** IDT-vectoring error code. */
210	uint32_t uIdtVectoringErrorCode;
211
212	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
213	uint32_t fVmcsFieldsRead;
214	/** Whether TSC-offsetting should be setup before VM-entry. */
215	bool fUpdateTscOffsettingAndPreemptTimer;
216	/** Whether the VM-exit was caused by a page-fault during delivery of a
217	* contributary exception or a page-fault. */
218	bool fVectoringPF;
219	} VMXTRANSIENT, *PVMXTRANSIENT;
220	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
221	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
222	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
223	/** @} */
224
225
226	/**
227	* MSR-bitmap read permissions.
228	*/
229	typedef enum VMXMSREXITREAD
230	{
231	/** Reading this MSR causes a VM-exit. */
232	VMXMSREXIT_INTERCEPT_READ = 0xb,
233	/** Reading this MSR does not cause a VM-exit. */
234	VMXMSREXIT_PASSTHRU_READ
235	} VMXMSREXITREAD;
236
237	/**
238	* MSR-bitmap write permissions.
239	*/
240	typedef enum VMXMSREXITWRITE
241	{
242	/** Writing to this MSR causes a VM-exit. */
243	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
244	/** Writing to this MSR does not cause a VM-exit. */
245	VMXMSREXIT_PASSTHRU_WRITE
246	} VMXMSREXITWRITE;
247
248
249	/*******************************************************************************
250	* Internal Functions *
251	*******************************************************************************/
252	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
253	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
254	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
255	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
256	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
257	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
258	#endif
259	#ifndef HMVMX_USE_FUNCTION_TABLE
260	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
261	#define HMVMX_EXIT_DECL static int
262	#else
263	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
264	#endif
265
266	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310
311	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
318
319
320	/*******************************************************************************
321	* Global Variables *
322	*******************************************************************************/
323	#ifdef HMVMX_USE_FUNCTION_TABLE
324	/**
325	* VM-exit handler.
326	*
327	* @returns VBox status code.
328	* @param pVCpu Pointer to the VMCPU.
329	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
330	* out-of-sync. Make sure to update the required
331	* fields before using them.
332	* @param pVmxTransient Pointer to the VMX-transient structure.
333	*/
334	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335	/** Pointer to VM-exit handler. */
336	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
337
338	/**
339	* VMX_EXIT dispatch table.
340	*/
341	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
342	{
343	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
344	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
345	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
346	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
347	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
348	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
349	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
350	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
351	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
352	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
353	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
354	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
355	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
356	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
357	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
358	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
359	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
360	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
361	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
362	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
363	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
364	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
365	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
366	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
367	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
368	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
369	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
370	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
371	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
372	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
373	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
374	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
375	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
376	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
377	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
378	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
379	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
380	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
381	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
382	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
383	/* 40 UNDEFINED */ hmR0VmxExitPause,
384	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
385	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
386	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
387	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
388	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
389	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
390	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
391	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
392	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
393	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
394	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
395	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
396	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
397	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
398	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
399	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
400	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
401	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
402	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
403	};
404	#endif /* HMVMX_USE_FUNCTION_TABLE */
405
406	#ifdef VBOX_STRICT
407	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
408	{
409	/* 0 */ "(Not Used)",
410	/* 1 */ "VMCALL executed in VMX root operation.",
411	/* 2 */ "VMCLEAR with invalid physical address.",
412	/* 3 */ "VMCLEAR with VMXON pointer.",
413	/* 4 */ "VMLAUNCH with non-clear VMCS.",
414	/* 5 */ "VMRESUME with non-launched VMCS.",
415	/* 6 */ "VMRESUME after VMXOFF",
416	/* 7 */ "VM entry with invalid control fields.",
417	/* 8 */ "VM entry with invalid host state fields.",
418	/* 9 */ "VMPTRLD with invalid physical address.",
419	/* 10 */ "VMPTRLD with VMXON pointer.",
420	/* 11 */ "VMPTRLD with incorrect revision identifier.",
421	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
422	/* 13 */ "VMWRITE to read-only VMCS component.",
423	/* 14 */ "(Not Used)",
424	/* 15 */ "VMXON executed in VMX root operation.",
425	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
426	/* 17 */ "VM entry with non-launched executing VMCS.",
427	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
428	/* 19 */ "VMCALL with non-clear VMCS.",
429	/* 20 */ "VMCALL with invalid VM-exit control fields.",
430	/* 21 */ "(Not Used)",
431	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
432	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
433	/* 24 */ "VMCALL with invalid SMM-monitor features.",
434	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
435	/* 26 */ "VM entry with events blocked by MOV SS.",
436	/* 27 */ "(Not Used)",
437	/* 28 */ "Invalid operand to INVEPT/INVVPID."
438	};
439	#endif /* VBOX_STRICT */
440
441
442
443	/**
444	* Updates the VM's last error record. If there was a VMX instruction error,
445	* reads the error data from the VMCS and updates VCPU's last error record as
446	* well.
447	*
448	* @param pVM Pointer to the VM.
449	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
450	* VERR_VMX_UNABLE_TO_START_VM or
451	* VERR_VMX_INVALID_VMCS_FIELD).
452	* @param rc The error code.
453	*/
454	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
455	{
456	AssertPtr(pVM);
457	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
458	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
459	{
460	AssertPtrReturnVoid(pVCpu);
461	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
462	}
463	pVM->hm.s.lLastError = rc;
464	}
465
466
467	/**
468	* Reads the VM-entry interruption-information field from the VMCS into the VMX
469	* transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVmxTransient Pointer to the VMX transient structure.
473	*
474	* @remarks No-long-jump zone!!!
475	*/
476	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
477	{
478	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
479	AssertRCReturn(rc, rc);
480	return VINF_SUCCESS;
481	}
482
483
484	/**
485	* Reads the VM-entry exception error code field from the VMCS into
486	* the VMX transient structure.
487	*
488	* @returns VBox status code.
489	* @param pVmxTransient Pointer to the VMX transient structure.
490	*
491	* @remarks No-long-jump zone!!!
492	*/
493	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
494	{
495	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
496	AssertRCReturn(rc, rc);
497	return VINF_SUCCESS;
498	}
499
500
501	/**
502	* Reads the VM-entry exception error code field from the VMCS into
503	* the VMX transient structure.
504	*
505	* @returns VBox status code.
506	* @param pVCpu Pointer to the VMCPU.
507	* @param pVmxTransient Pointer to the VMX transient structure.
508	*
509	* @remarks No-long-jump zone!!!
510	*/
511	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
512	{
513	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
514	AssertRCReturn(rc, rc);
515	return VINF_SUCCESS;
516	}
517
518
519	/**
520	* Reads the VM-exit interruption-information field from the VMCS into the VMX
521	* transient structure.
522	*
523	* @returns VBox status code.
524	* @param pVCpu Pointer to the VMCPU.
525	* @param pVmxTransient Pointer to the VMX transient structure.
526	*/
527	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
528	{
529	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
530	{
531	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
532	AssertRCReturn(rc, rc);
533	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
534	}
535	return VINF_SUCCESS;
536	}
537
538
539	/**
540	* Reads the VM-exit interruption error code from the VMCS into the VMX
541	* transient structure.
542	*
543	* @returns VBox status code.
544	* @param pVCpu Pointer to the VMCPU.
545	* @param pVmxTransient Pointer to the VMX transient structure.
546	*/
547	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
548	{
549	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
550	{
551	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
552	AssertRCReturn(rc, rc);
553	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
554	}
555	return VINF_SUCCESS;
556	}
557
558
559	/**
560	* Reads the VM-exit instruction length field from the VMCS into the VMX
561	* transient structure.
562	*
563	* @returns VBox status code.
564	* @param pVCpu Pointer to the VMCPU.
565	* @param pVmxTransient Pointer to the VMX transient structure.
566	*/
567	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
568	{
569	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
570	{
571	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
572	AssertRCReturn(rc, rc);
573	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
574	}
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the exit qualification from the VMCS into the VMX transient structure.
581	*
582	* @returns VBox status code.
583	* @param pVCpu Pointer to the VMCPU.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*/
586	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
587	{
588	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
589	{
590	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
591	AssertRCReturn(rc, rc);
592	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
593	}
594	return VINF_SUCCESS;
595	}
596
597
598	/**
599	* Reads the IDT-vectoring information field from the VMCS into the VMX
600	* transient structure.
601	*
602	* @returns VBox status code.
603	* @param pVmxTransient Pointer to the VMX transient structure.
604	*
605	* @remarks No-long-jump zone!!!
606	*/
607	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
608	{
609	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
612	AssertRCReturn(rc, rc);
613	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
614	}
615	return VINF_SUCCESS;
616	}
617
618
619	/**
620	* Reads the IDT-vectoring error code from the VMCS into the VMX
621	* transient structure.
622	*
623	* @returns VBox status code.
624	* @param pVmxTransient Pointer to the VMX transient structure.
625	*/
626	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627	{
628	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
629	{
630	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
631	AssertRCReturn(rc, rc);
632	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
633	}
634	return VINF_SUCCESS;
635	}
636
637
638	/**
639	* Enters VMX root mode operation on the current CPU.
640	*
641	* @returns VBox status code.
642	* @param pVM Pointer to the VM (optional, can be NULL, after
643	* a resume).
644	* @param HCPhysCpuPage Physical address of the VMXON region.
645	* @param pvCpuPage Pointer to the VMXON region.
646	*/
647	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
648	{
649	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
650	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
651	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
652
653	if (pVM)
654	{
655	/* Write the VMCS revision dword to the VMXON region. */
656	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
657	}
658
659	/* Enable the VMX bit in CR4 if necessary. */
660	RTCCUINTREG uCr4 = ASMGetCR4();
661	if (!(uCr4 & X86_CR4_VMXE))
662	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
663
664	/* Enter VMX root mode. */
665	int rc = VMXEnable(HCPhysCpuPage);
666	if (RT_FAILURE(rc))
667	ASMSetCR4(uCr4);
668
669	return rc;
670	}
671
672
673	/**
674	* Exits VMX root mode operation on the current CPU.
675	*
676	* @returns VBox status code.
677	*/
678	static int hmR0VmxLeaveRootMode(void)
679	{
680	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
681
682	/* If we're for some reason not in VMX root mode, then don't leave it. */
683	RTCCUINTREG uHostCR4 = ASMGetCR4();
684	if (uHostCR4 & X86_CR4_VMXE)
685	{
686	/* Exit VMX root mode and clear the VMX bit in CR4. */
687	VMXDisable();
688	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
689	return VINF_SUCCESS;
690	}
691
692	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
693	}
694
695
696	/**
697	* Allocates and maps one physically contiguous page. The allocated page is
698	* zero'd out. (Used by various VT-x structures).
699	*
700	* @returns IPRT status code.
701	* @param pMemObj Pointer to the ring-0 memory object.
702	* @param ppVirt Where to store the virtual address of the
703	* allocation.
704	* @param pPhys Where to store the physical address of the
705	* allocation.
706	*/
707	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
708	{
709	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
710	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
711	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
712
713	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
714	if (RT_FAILURE(rc))
715	return rc;
716	ppVirt = RTR0MemObjAddress(pMemObj);
717	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
718	ASMMemZero32(*ppVirt, PAGE_SIZE);
719	return VINF_SUCCESS;
720	}
721
722
723	/**
724	* Frees and unmaps an allocated physical page.
725	*
726	* @param pMemObj Pointer to the ring-0 memory object.
727	* @param ppVirt Where to re-initialize the virtual address of
728	* allocation as 0.
729	* @param pHCPhys Where to re-initialize the physical address of the
730	* allocation as 0.
731	*/
732	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
733	{
734	AssertPtr(pMemObj);
735	AssertPtr(ppVirt);
736	AssertPtr(pHCPhys);
737	if (*pMemObj != NIL_RTR0MEMOBJ)
738	{
739	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
740	AssertRC(rc);
741	*pMemObj = NIL_RTR0MEMOBJ;
742	*ppVirt = 0;
743	*pHCPhys = 0;
744	}
745	}
746
747
748	/**
749	* Worker function to free VT-x related structures.
750	*
751	* @returns IPRT status code.
752	* @param pVM Pointer to the VM.
753	*/
754	static void hmR0VmxStructsFree(PVM pVM)
755	{
756	for (VMCPUID i = 0; i < pVM->cCpus; i++)
757	{
758	PVMCPU pVCpu = &pVM->aCpus[i];
759	AssertPtr(pVCpu);
760
761	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
763	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
764	#endif
765
766	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
768
769	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
770	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
771	}
772
773	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
774	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
775	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
776	#endif
777	}
778
779
780	/**
781	* Worker function to allocate VT-x related VM structures.
782	*
783	* @returns IPRT status code.
784	* @param pVM Pointer to the VM.
785	*/
786	static int hmR0VmxStructsAlloc(PVM pVM)
787	{
788	/*
789	* Initialize members up-front so we can cleanup properly on allocation failure.
790	*/
791	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
792	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
793	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
794	pVM->hm.s.vmx.HCPhys##a_Name = 0;
795
796	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
797	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
798	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
799	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
800
801	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
802	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
803	#endif
804	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
805
806	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
807	for (VMCPUID i = 0; i < pVM->cCpus; i++)
808	{
809	PVMCPU pVCpu = &pVM->aCpus[i];
810	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
813	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
814	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
815	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
816	#endif
817	}
818	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
819	#undef VMXLOCAL_INIT_VM_MEMOBJ
820
821	/*
822	* Allocate all the VT-x structures.
823	*/
824	int rc = VINF_SUCCESS;
825	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
826	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
827	if (RT_FAILURE(rc))
828	goto cleanup;
829	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
830	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
831	#endif
832
833	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
834	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
835	{
836	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
837	&pVM->hm.s.vmx.HCPhysApicAccess);
838	if (RT_FAILURE(rc))
839	goto cleanup;
840	}
841
842	/*
843	* Initialize per-VCPU VT-x structures.
844	*/
845	for (VMCPUID i = 0; i < pVM->cCpus; i++)
846	{
847	PVMCPU pVCpu = &pVM->aCpus[i];
848	AssertPtr(pVCpu);
849
850	/* Allocate the VM control structure (VMCS). */
851	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
852	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
853	if (RT_FAILURE(rc))
854	goto cleanup;
855
856	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
857	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
858	{
859	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
860	&pVCpu->hm.s.vmx.HCPhysVirtApic);
861	if (RT_FAILURE(rc))
862	goto cleanup;
863	}
864
865	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
866	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
867	{
868	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
869	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
870	if (RT_FAILURE(rc))
871	goto cleanup;
872	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
873	}
874
875	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
876	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
877	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878	if (RT_FAILURE(rc))
879	goto cleanup;
880
881	/* Allocate the VM-exit MSR-load page for the host MSRs. */
882	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
883	if (RT_FAILURE(rc))
884	goto cleanup;
885	#endif
886	}
887
888	return VINF_SUCCESS;
889
890	cleanup:
891	hmR0VmxStructsFree(pVM);
892	return rc;
893	}
894
895
896	/**
897	* Does global VT-x initialization (called during module initialization).
898	*
899	* @returns VBox status code.
900	*/
901	VMMR0DECL(int) VMXR0GlobalInit(void)
902	{
903	#ifdef HMVMX_USE_FUNCTION_TABLE
904	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
905	# ifdef VBOX_STRICT
906	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
907	Assert(g_apfnVMExitHandlers[i]);
908	# endif
909	#endif
910	return VINF_SUCCESS;
911	}
912
913
914	/**
915	* Does global VT-x termination (called during module termination).
916	*/
917	VMMR0DECL(void) VMXR0GlobalTerm()
918	{
919	/* Nothing to do currently. */
920	}
921
922
923	/**
924	* Sets up and activates VT-x on the current CPU.
925	*
926	* @returns VBox status code.
927	* @param pCpu Pointer to the global CPU info struct.
928	* @param pVM Pointer to the VM (can be NULL after a host resume
929	* operation).
930	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
931	* fEnabledByHost is true).
932	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
933	* @a fEnabledByHost is true).
934	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
935	* enable VT-x on the host.
936	*/
937	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
938	{
939	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
940	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
941
942	if (!fEnabledByHost)
943	{
944	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
945	if (RT_FAILURE(rc))
946	return rc;
947	}
948
949	/*
950	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
951	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
952	* each time while reusing a VPID after hitting the MaxASID limit once.
953	*/
954	if ( pVM
955	&& pVM->hm.s.fNestedPaging)
956	{
957	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
958	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
959	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
960	pCpu->fFlushAsidBeforeUse = false;
961	}
962	else
963	{
964	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
965	* without Nested Paging triggered this function) we still have the risk
966	* of potentially running with stale TLB-entries from other hypervisors
967	* when later we use a VM with NestedPaging. To fix this properly we will
968	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
969	* 'vmx_ept_vpid_caps' from it. Sigh. */
970	pCpu->fFlushAsidBeforeUse = true;
971	}
972
973	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
974	++pCpu->cTlbFlushes;
975
976	return VINF_SUCCESS;
977	}
978
979
980	/**
981	* Deactivates VT-x on the current CPU.
982	*
983	* @returns VBox status code.
984	* @param pCpu Pointer to the global CPU info struct.
985	* @param pvCpuPage Pointer to the VMXON region.
986	* @param HCPhysCpuPage Physical address of the VMXON region.
987	*
988	* @remarks This function should never be called when SUPR0EnableVTx() or
989	* similar was used to enable VT-x on the host.
990	*/
991	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
992	{
993	NOREF(pCpu);
994	NOREF(pvCpuPage);
995	NOREF(HCPhysCpuPage);
996
997	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
998	return hmR0VmxLeaveRootMode();
999	}
1000
1001
1002	/**
1003	* Sets the permission bits for the specified MSR in the MSR bitmap.
1004	*
1005	* @param pVCpu Pointer to the VMCPU.
1006	* @param uMSR The MSR value.
1007	* @param enmRead Whether reading this MSR causes a VM-exit.
1008	* @param enmWrite Whether writing this MSR causes a VM-exit.
1009	*/
1010	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1011	{
1012	int32_t iBit;
1013	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1014
1015	/*
1016	* Layout:
1017	* 0x000 - 0x3ff - Low MSR read bits
1018	* 0x400 - 0x7ff - High MSR read bits
1019	* 0x800 - 0xbff - Low MSR write bits
1020	* 0xc00 - 0xfff - High MSR write bits
1021	*/
1022	if (uMsr <= 0x00001FFF)
1023	iBit = uMsr;
1024	else if ( uMsr >= 0xC0000000
1025	&& uMsr <= 0xC0001FFF)
1026	{
1027	iBit = (uMsr - 0xC0000000);
1028	pbMsrBitmap += 0x400;
1029	}
1030	else
1031	{
1032	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1033	return;
1034	}
1035
1036	Assert(iBit <= 0x1fff);
1037	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1038	ASMBitSet(pbMsrBitmap, iBit);
1039	else
1040	ASMBitClear(pbMsrBitmap, iBit);
1041
1042	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1043	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1044	else
1045	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1046	}
1047
1048
1049	/**
1050	* Flushes the TLB using EPT.
1051	*
1052	* @returns VBox status code.
1053	* @param pVM Pointer to the VM.
1054	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1055	* enmFlush).
1056	* @param enmFlush Type of flush.
1057	*/
1058	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1059	{
1060	AssertPtr(pVM);
1061	Assert(pVM->hm.s.fNestedPaging);
1062
1063	uint64_t descriptor[2];
1064	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1065	descriptor[0] = 0;
1066	else
1067	{
1068	Assert(pVCpu);
1069	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1070	}
1071	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1072
1073	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1074	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1075	rc));
1076	if ( RT_SUCCESS(rc)
1077	&& pVCpu)
1078	{
1079	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1080	}
1081	}
1082
1083
1084	/**
1085	* Flushes the TLB using VPID.
1086	*
1087	* @returns VBox status code.
1088	* @param pVM Pointer to the VM.
1089	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1090	* enmFlush).
1091	* @param enmFlush Type of flush.
1092	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1093	* on @a enmFlush).
1094	*/
1095	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1096	{
1097	AssertPtr(pVM);
1098	Assert(pVM->hm.s.vmx.fVpid);
1099
1100	uint64_t descriptor[2];
1101	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1102	{
1103	descriptor[0] = 0;
1104	descriptor[1] = 0;
1105	}
1106	else
1107	{
1108	AssertPtr(pVCpu);
1109	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1110	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1111	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1112	descriptor[1] = GCPtr;
1113	}
1114
1115	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1116	AssertMsg(rc == VINF_SUCCESS,
1117	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1118	if ( RT_SUCCESS(rc)
1119	&& pVCpu)
1120	{
1121	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1122	}
1123	}
1124
1125
1126	/**
1127	* Invalidates a guest page by guest virtual address. Only relevant for
1128	* EPT/VPID, otherwise there is nothing really to invalidate.
1129	*
1130	* @returns VBox status code.
1131	* @param pVM Pointer to the VM.
1132	* @param pVCpu Pointer to the VMCPU.
1133	* @param GCVirt Guest virtual address of the page to invalidate.
1134	*/
1135	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1136	{
1137	AssertPtr(pVM);
1138	AssertPtr(pVCpu);
1139	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1140
1141	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	if (!fFlushPending)
1143	{
1144	/*
1145	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1146	* See @bugref{6043} and @bugref{6177}.
1147	*
1148	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1149	* function maybe called in a loop with individual addresses.
1150	*/
1151	if (pVM->hm.s.vmx.fVpid)
1152	{
1153	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1154	{
1155	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1156	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1157	}
1158	else
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	}
1161	else if (pVM->hm.s.fNestedPaging)
1162	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1163	}
1164
1165	return VINF_SUCCESS;
1166	}
1167
1168
1169	/**
1170	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1171	* otherwise there is nothing really to invalidate.
1172	*
1173	* @returns VBox status code.
1174	* @param pVM Pointer to the VM.
1175	* @param pVCpu Pointer to the VMCPU.
1176	* @param GCPhys Guest physical address of the page to invalidate.
1177	*/
1178	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1179	{
1180	LogFlowFunc(("%RGp\n", GCPhys));
1181
1182	/*
1183	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1184	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1185	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1186	*/
1187	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1188	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1189	return VINF_SUCCESS;
1190	}
1191
1192
1193	/**
1194	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1195	* case where neither EPT nor VPID is supported by the CPU.
1196	*
1197	* @param pVM Pointer to the VM.
1198	* @param pVCpu Pointer to the VMCPU.
1199	*
1200	* @remarks Called with interrupts disabled.
1201	*/
1202	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1203	{
1204	NOREF(pVM);
1205	AssertPtr(pVCpu);
1206	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1207	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1208
1209	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1210	AssertPtr(pCpu);
1211
1212	pVCpu->hm.s.TlbShootdown.cPages = 0;
1213	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1214	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1215	pVCpu->hm.s.fForceTLBFlush = false;
1216	return;
1217	}
1218
1219
1220	/**
1221	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1222	*
1223	* @param pVM Pointer to the VM.
1224	* @param pVCpu Pointer to the VMCPU.
1225	* @remarks All references to "ASID" in this function pertains to "VPID" in
1226	* Intel's nomenclature. The reason is, to avoid confusion in compare
1227	* statements since the host-CPU copies are named "ASID".
1228	*
1229	* @remarks Called with interrupts disabled.
1230	*/
1231	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1232	{
1233	#ifdef VBOX_WITH_STATISTICS
1234	bool fTlbFlushed = false;
1235	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1236	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1237	if (!fTlbFlushed) \
1238	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1239	} while (0)
1240	#else
1241	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1242	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1243	#endif
1244
1245	AssertPtr(pVM);
1246	AssertPtr(pVCpu);
1247	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1248	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1249	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1250
1251	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1252	AssertPtr(pCpu);
1253
1254	/*
1255	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1256	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1257	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1258	*/
1259	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1260	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1261	{
1262	++pCpu->uCurrentAsid;
1263	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1264	{
1265	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1266	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1267	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1268	}
1269
1270	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1271	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1272	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1273
1274	/*
1275	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1276	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1277	*/
1278	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1279	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1280	HMVMX_SET_TAGGED_TLB_FLUSHED();
1281	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1282	}
1283
1284	/* Check for explicit TLB shootdowns. */
1285	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1286	{
1287	/*
1288	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1289	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1290	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1291	* but not guest-physical mappings.
1292	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1293	*/
1294	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1295	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1296	HMVMX_SET_TAGGED_TLB_FLUSHED();
1297	}
1298
1299	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1300	* not be executed. See hmQueueInvlPage() where it is commented
1301	* out. Support individual entry flushing someday. */
1302	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1303	{
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1305
1306	/*
1307	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1308	* as supported by the CPU.
1309	*/
1310	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1311	{
1312	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1313	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1314	}
1315	else
1316	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1317
1318	HMVMX_SET_TAGGED_TLB_FLUSHED();
1319	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1320	}
1321
1322	pVCpu->hm.s.TlbShootdown.cPages = 0;
1323	pVCpu->hm.s.fForceTLBFlush = false;
1324
1325	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1326
1327	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1328	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1329	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1330	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1331	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1332	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1333	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1334	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1335
1336	/* Update VMCS with the VPID. */
1337	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1338	AssertRC(rc);
1339
1340	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1341	}
1342
1343
1344	/**
1345	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1346	*
1347	* @returns VBox status code.
1348	* @param pVM Pointer to the VM.
1349	* @param pVCpu Pointer to the VMCPU.
1350	*
1351	* @remarks Called with interrupts disabled.
1352	*/
1353	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1354	{
1355	AssertPtr(pVM);
1356	AssertPtr(pVCpu);
1357	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1358	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1359
1360	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1361	AssertPtr(pCpu);
1362
1363	/*
1364	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1365	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1366	*/
1367	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1368	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1369	{
1370	pVCpu->hm.s.fForceTLBFlush = true;
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1372	}
1373
1374	/* Check for explicit TLB shootdown flushes. */
1375	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1376	{
1377	pVCpu->hm.s.fForceTLBFlush = true;
1378	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1379	}
1380
1381	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1382	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1383
1384	if (pVCpu->hm.s.fForceTLBFlush)
1385	{
1386	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1387	pVCpu->hm.s.fForceTLBFlush = false;
1388	}
1389	else
1390	{
1391	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1392	* not be executed. See hmQueueInvlPage() where it is commented
1393	* out. Support individual entry flushing someday. */
1394	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1395	{
1396	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1397	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1398	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1399	}
1400	else
1401	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1402	}
1403
1404	pVCpu->hm.s.TlbShootdown.cPages = 0;
1405	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1406	}
1407
1408
1409	/**
1410	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1411	*
1412	* @returns VBox status code.
1413	* @param pVM Pointer to the VM.
1414	* @param pVCpu Pointer to the VMCPU.
1415	*
1416	* @remarks Called with interrupts disabled.
1417	*/
1418	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1419	{
1420	AssertPtr(pVM);
1421	AssertPtr(pVCpu);
1422	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1423	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1424
1425	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1426
1427	/*
1428	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1429	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1430	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1431	*/
1432	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1433	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1434	{
1435	pVCpu->hm.s.fForceTLBFlush = true;
1436	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1437	}
1438
1439	/* Check for explicit TLB shootdown flushes. */
1440	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1441	{
1442	/*
1443	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1444	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1445	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1446	*/
1447	pVCpu->hm.s.fForceTLBFlush = true;
1448	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1449	}
1450
1451	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1452	if (pVCpu->hm.s.fForceTLBFlush)
1453	{
1454	++pCpu->uCurrentAsid;
1455	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1456	{
1457	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1458	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1459	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1460	}
1461
1462	pVCpu->hm.s.fForceTLBFlush = false;
1463	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1464	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1465	if (pCpu->fFlushAsidBeforeUse)
1466	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1467	}
1468	else
1469	{
1470	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1471	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1472	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1473	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1474
1475	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1476	* not be executed. See hmQueueInvlPage() where it is commented
1477	* out. Support individual entry flushing someday. */
1478	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1479	{
1480	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1481	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1482	{
1483	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1484	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1485	}
1486	else
1487	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1488	}
1489	else
1490	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1491	}
1492
1493	pVCpu->hm.s.TlbShootdown.cPages = 0;
1494	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1495
1496	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1497	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1498	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1499	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1500	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1501	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1502
1503	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1504	AssertRC(rc);
1505	}
1506
1507
1508	/**
1509	* Flushes the guest TLB entry based on CPU capabilities.
1510	*
1511	* @param pVCpu Pointer to the VMCPU.
1512	*/
1513	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1514	{
1515	PVM pVM = pVCpu->CTX_SUFF(pVM);
1516	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1517	{
1518	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1519	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1520	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1521	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1522	default:
1523	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1524	break;
1525	}
1526	}
1527
1528
1529	/**
1530	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1531	* TLB entries from the host TLB before VM-entry.
1532	*
1533	* @returns VBox status code.
1534	* @param pVM Pointer to the VM.
1535	*/
1536	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1537	{
1538	/*
1539	* Determine optimal flush type for Nested Paging.
1540	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1541	* guest execution (see hmR3InitFinalizeR0()).
1542	*/
1543	if (pVM->hm.s.fNestedPaging)
1544	{
1545	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1546	{
1547	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1548	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1549	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1550	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1551	else
1552	{
1553	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1554	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1555	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1556	}
1557
1558	/* Make sure the write-back cacheable memory type for EPT is supported. */
1559	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1560	{
1561	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1562	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1563	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1564	}
1565	}
1566	else
1567	{
1568	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1569	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1570	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1571	}
1572	}
1573
1574	/*
1575	* Determine optimal flush type for VPID.
1576	*/
1577	if (pVM->hm.s.vmx.fVpid)
1578	{
1579	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1580	{
1581	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1582	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1583	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1584	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1585	else
1586	{
1587	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1588	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1589	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1590	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1591	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1592	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1593	pVM->hm.s.vmx.fVpid = false;
1594	}
1595	}
1596	else
1597	{
1598	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1599	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1600	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1601	pVM->hm.s.vmx.fVpid = false;
1602	}
1603	}
1604
1605	/*
1606	* Setup the handler for flushing tagged-TLBs.
1607	*/
1608	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1609	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1610	else if (pVM->hm.s.fNestedPaging)
1611	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1612	else if (pVM->hm.s.vmx.fVpid)
1613	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1614	else
1615	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1616	return VINF_SUCCESS;
1617	}
1618
1619
1620	/**
1621	* Sets up pin-based VM-execution controls in the VMCS.
1622	*
1623	* @returns VBox status code.
1624	* @param pVM Pointer to the VM.
1625	* @param pVCpu Pointer to the VMCPU.
1626	*/
1627	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1628	{
1629	AssertPtr(pVM);
1630	AssertPtr(pVCpu);
1631
1632	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1633	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1634
1635	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1636	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1637	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1638
1639	/* Enable the VMX preemption timer. */
1640	if (pVM->hm.s.vmx.fUsePreemptTimer)
1641	{
1642	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1643	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1644	}
1645
1646	if ((val & zap) != val)
1647	{
1648	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1649	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1654	AssertRCReturn(rc, rc);
1655
1656	/* Update VCPU with the currently set pin-based VM-execution controls. */
1657	pVCpu->hm.s.vmx.u32PinCtls = val;
1658	return rc;
1659	}
1660
1661
1662	/**
1663	* Sets up processor-based VM-execution controls in the VMCS.
1664	*
1665	* @returns VBox status code.
1666	* @param pVM Pointer to the VM.
1667	* @param pVMCPU Pointer to the VMCPU.
1668	*/
1669	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1670	{
1671	AssertPtr(pVM);
1672	AssertPtr(pVCpu);
1673
1674	int rc = VERR_INTERNAL_ERROR_5;
1675	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1676	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1677
1678	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1679	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1681	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1682	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1683	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1684	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1685
1686	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1687	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1688	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1689	{
1690	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1691	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1692	}
1693
1694	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1695	if (!pVM->hm.s.fNestedPaging)
1696	{
1697	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1698	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1699	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1700	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1701	}
1702
1703	/* Use TPR shadowing if supported by the CPU. */
1704	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1705	{
1706	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1707	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1708	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1709	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1710	AssertRCReturn(rc, rc);
1711
1712	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1713	/* CR8 writes causes a VM-exit based on TPR threshold. */
1714	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1715	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1716	}
1717	else
1718	{
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1720	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1721	}
1722
1723	/* Use MSR-bitmaps if supported by the CPU. */
1724	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1725	{
1726	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1727
1728	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1729	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1730	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1731	AssertRCReturn(rc, rc);
1732
1733	/*
1734	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1735	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1736	*/
1737	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1738	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1739	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1740	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1743	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1744	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1745	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1746	}
1747
1748	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1749	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1751
1752	if ((val & zap) != val)
1753	{
1754	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1755	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1756	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1757	}
1758
1759	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1760	AssertRCReturn(rc, rc);
1761
1762	/* Update VCPU with the currently set processor-based VM-execution controls. */
1763	pVCpu->hm.s.vmx.u32ProcCtls = val;
1764
1765	/*
1766	* Secondary processor-based VM-execution controls.
1767	*/
1768	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1769	{
1770	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1771	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1772
1773	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1774	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1775
1776	if (pVM->hm.s.fNestedPaging)
1777	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1778	else
1779	{
1780	/*
1781	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1782	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1783	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1784	*/
1785	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1786	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1787	}
1788
1789	if (pVM->hm.s.vmx.fVpid)
1790	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1791
1792	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1793	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1794
1795	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1796	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1797	* done dynamically. */
1798	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1799	{
1800	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1801	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1802	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1803	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1804	AssertRCReturn(rc, rc);
1805	}
1806
1807	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1808	{
1809	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1810	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1811	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812	}
1813
1814	if ((val & zap) != val)
1815	{
1816	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1817	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1818	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1819	}
1820
1821	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1822	AssertRCReturn(rc, rc);
1823
1824	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1825	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1826	}
1827
1828	return VINF_SUCCESS;
1829	}
1830
1831
1832	/**
1833	* Sets up miscellaneous (everything other than Pin & Processor-based
1834	* VM-execution) control fields in the VMCS.
1835	*
1836	* @returns VBox status code.
1837	* @param pVM Pointer to the VM.
1838	* @param pVCpu Pointer to the VMCPU.
1839	*/
1840	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1841	{
1842	AssertPtr(pVM);
1843	AssertPtr(pVCpu);
1844
1845	int rc = VERR_GENERAL_FAILURE;
1846
1847	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1848	#if 0
1849	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1850	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1851	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1852
1853	/*
1854	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1855	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1856	* We thus use the exception bitmap to control it rather than use both.
1857	*/
1858	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1859	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1860
1861	/** @todo Explore possibility of using IO-bitmaps. */
1862	/* All IO & IOIO instructions cause VM-exits. */
1863	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1864	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1865
1866	/* Initialize the MSR-bitmap area. */
1867	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1868	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1869	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1870	#endif
1871
1872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1873	/* Setup MSR autoloading/storing. */
1874	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1875	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1876	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1877	AssertRCReturn(rc, rc);
1878	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1879	AssertRCReturn(rc, rc);
1880
1881	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1882	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1883	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1884	AssertRCReturn(rc, rc);
1885	#endif
1886
1887	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1888	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1889	AssertRCReturn(rc, rc);
1890
1891	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1892	#if 0
1893	/* Setup debug controls */
1894	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1895	AssertRCReturn(rc, rc);
1896	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1897	AssertRCReturn(rc, rc);
1898	#endif
1899
1900	return rc;
1901	}
1902
1903
1904	/**
1905	* Sets up the initial exception bitmap in the VMCS based on static conditions
1906	* (i.e. conditions that cannot ever change at runtime).
1907	*
1908	* @returns VBox status code.
1909	* @param pVM Pointer to the VM.
1910	* @param pVCpu Pointer to the VMCPU.
1911	*/
1912	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1913	{
1914	AssertPtr(pVM);
1915	AssertPtr(pVCpu);
1916
1917	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1918
1919	uint32_t u32XcptBitmap = 0;
1920
1921	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1922	if (!pVM->hm.s.fNestedPaging)
1923	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1924
1925	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1926	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1927	AssertRCReturn(rc, rc);
1928	return rc;
1929	}
1930
1931
1932	/**
1933	* Sets up the initial guest-state mask. The guest-state mask is consulted
1934	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1935	* for the nested virtualization case (as it would cause a VM-exit).
1936	*
1937	* @param pVCpu Pointer to the VMCPU.
1938	*/
1939	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1940	{
1941	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1942	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1943	return VINF_SUCCESS;
1944	}
1945
1946
1947	/**
1948	* Does per-VM VT-x initialization.
1949	*
1950	* @returns VBox status code.
1951	* @param pVM Pointer to the VM.
1952	*/
1953	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1954	{
1955	LogFlowFunc(("pVM=%p\n", pVM));
1956
1957	int rc = hmR0VmxStructsAlloc(pVM);
1958	if (RT_FAILURE(rc))
1959	{
1960	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1961	return rc;
1962	}
1963
1964	return VINF_SUCCESS;
1965	}
1966
1967
1968	/**
1969	* Does per-VM VT-x termination.
1970	*
1971	* @returns VBox status code.
1972	* @param pVM Pointer to the VM.
1973	*/
1974	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1975	{
1976	LogFlowFunc(("pVM=%p\n", pVM));
1977
1978	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1979	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1980	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1981	#endif
1982	hmR0VmxStructsFree(pVM);
1983	return VINF_SUCCESS;
1984	}
1985
1986
1987	/**
1988	* Sets up the VM for execution under VT-x.
1989	* This function is only called once per-VM during initalization.
1990	*
1991	* @returns VBox status code.
1992	* @param pVM Pointer to the VM.
1993	*/
1994	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1995	{
1996	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1997	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1998
1999	LogFlowFunc(("pVM=%p\n", pVM));
2000
2001	/*
2002	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2003	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2004	*/
2005	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2006	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2007	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2008	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2009	{
2010	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2011	return VERR_INTERNAL_ERROR;
2012	}
2013
2014	/* Initialize these always, see hmR3InitFinalizeR0().*/
2015	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2016	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2017
2018	/* Setup the tagged-TLB flush handlers. */
2019	int rc = hmR0VmxSetupTaggedTlb(pVM);
2020	if (RT_FAILURE(rc))
2021	{
2022	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2023	return rc;
2024	}
2025
2026	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2027	{
2028	PVMCPU pVCpu = &pVM->aCpus[i];
2029	AssertPtr(pVCpu);
2030	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2031
2032	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2033	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2034
2035	/* Set revision dword at the beginning of the VMCS structure. */
2036	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2037
2038	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2039	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2040	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2041	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2042
2043	/* Load this VMCS as the current VMCS. */
2044	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2045	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2046	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2047
2048	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2049	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2050	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2051
2052	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2053	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2054	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2055
2056	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2057	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2058	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2059
2060	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2061	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2062	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2063
2064	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2065	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2066	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2067
2068	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2069	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2070	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2071	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2072	#endif
2073
2074	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2075	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2076	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2077	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2078
2079	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2080	}
2081
2082	return VINF_SUCCESS;
2083	}
2084
2085
2086	/**
2087	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2088	* the VMCS.
2089	*
2090	* @returns VBox status code.
2091	* @param pVM Pointer to the VM.
2092	* @param pVCpu Pointer to the VMCPU.
2093	*/
2094	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2095	{
2096	RTCCUINTREG uReg = ASMGetCR0();
2097	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2098	AssertRCReturn(rc, rc);
2099
2100	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2101	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2102	if (HMVMX_IS_64BIT_HOST_MODE())
2103	{
2104	uint64_t uRegCR3 = HMR0Get64bitCR3();
2105	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2106	}
2107	else
2108	#endif
2109	{
2110	uReg = ASMGetCR3();
2111	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2112	}
2113	AssertRCReturn(rc, rc);
2114
2115	uReg = ASMGetCR4();
2116	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2117	AssertRCReturn(rc, rc);
2118	return rc;
2119	}
2120
2121
2122	/**
2123	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2124	* the host-state area in the VMCS.
2125	*
2126	* @returns VBox status code.
2127	* @param pVM Pointer to the VM.
2128	* @param pVCpu Pointer to the VMCPU.
2129	*/
2130	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2131	{
2132	int rc = VERR_INTERNAL_ERROR_5;
2133	RTSEL uSelDS = 0;
2134	RTSEL uSelES = 0;
2135	RTSEL uSelFS = 0;
2136	RTSEL uSelGS = 0;
2137	RTSEL uSelTR = 0;
2138
2139	/*
2140	* Host DS, ES, FS and GS segment registers.
2141	*/
2142	#if HC_ARCH_BITS == 64
2143	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2144	uSelDS = ASMGetDS();
2145	uSelES = ASMGetES();
2146	uSelFS = ASMGetFS();
2147	uSelGS = ASMGetGS();
2148	#endif
2149
2150	/*
2151	* Host CS and SS segment registers.
2152	*/
2153	RTSEL uSelCS;
2154	RTSEL uSelSS;
2155	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2156	if (HMVMX_IS_64BIT_HOST_MODE())
2157	{
2158	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2159	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2160	}
2161	else
2162	{
2163	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2164	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2165	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2166	}
2167	#else
2168	uSelCS = ASMGetCS();
2169	uSelSS = ASMGetSS();
2170	#endif
2171
2172	/*
2173	* Host TR segment register.
2174	*/
2175	uSelTR = ASMGetTR();
2176
2177	#if HC_ARCH_BITS == 64
2178	/*
2179	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2180	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2181	*/
2182	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2183	{
2184	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2185	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2186	uSelDS = 0;
2187	}
2188	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2189	{
2190	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2191	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2192	uSelES = 0;
2193	}
2194	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2195	{
2196	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2197	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2198	uSelFS = 0;
2199	}
2200	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2201	{
2202	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2203	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2204	uSelGS = 0;
2205	}
2206	#endif
2207
2208	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2209	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2210	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2211	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2212	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2213	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2214	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2215	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2216	Assert(uSelCS);
2217	Assert(uSelTR);
2218
2219	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2220	#if 0
2221	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2222	Assert(uSelSS != 0);
2223	#endif
2224
2225	/* Write these host selector fields into the host-state area in the VMCS. */
2226	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2227	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2228	#if HC_ARCH_BITS == 64
2229	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2230	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2231	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2232	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2233	#endif
2234	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2235
2236	/*
2237	* Host GDTR and IDTR.
2238	*/
2239	RTGDTR Gdtr;
2240	RT_ZERO(Gdtr);
2241	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2242	if (HMVMX_IS_64BIT_HOST_MODE())
2243	{
2244	X86XDTR64 Gdtr64;
2245	X86XDTR64 Idtr64;
2246	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2247	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2248	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2249
2250	Gdtr.cbGdt = Gdtr64.cb;
2251	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2252	}
2253	else
2254	#endif
2255	{
2256	RTIDTR Idtr;
2257	ASMGetGDTR(&Gdtr);
2258	ASMGetIDTR(&Idtr);
2259	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2260	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2261
2262	#if HC_ARCH_BITS == 64
2263	/*
2264	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2265	* maximum limit (0xffff) on every VM-exit.
2266	*/
2267	if (Gdtr.cbGdt != 0xffff)
2268	{
2269	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2270	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2271	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2272	}
2273
2274	/*
2275	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2276	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2277	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2278	*/
2279	if (Idtr.cbIdt < 0x0fff)
2280	{
2281	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2282	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2283	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2284	}
2285	#endif
2286	}
2287
2288	/*
2289	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2290	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2291	*/
2292	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2293	{
2294	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2295	return VERR_VMX_INVALID_HOST_STATE;
2296	}
2297
2298	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2299	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2300	if (HMVMX_IS_64BIT_HOST_MODE())
2301	{
2302	/* We need the 64-bit TR base for hybrid darwin. */
2303	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2304	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2305	}
2306	else
2307	#endif
2308	{
2309	uintptr_t uTRBase;
2310	#if HC_ARCH_BITS == 64
2311	uTRBase = X86DESC64_BASE(pDesc);
2312	#else
2313	uTRBase = X86DESC_BASE(pDesc);
2314	#endif
2315	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2316	}
2317	AssertRCReturn(rc, rc);
2318
2319	/*
2320	* Host FS base and GS base.
2321	*/
2322	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2323	if (HMVMX_IS_64BIT_HOST_MODE())
2324	{
2325	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2326	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2327	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2328	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2329
2330	# if HC_ARCH_BITS == 64
2331	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2332	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2333	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2334	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2335	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2336	# endif
2337	}
2338	#endif
2339	return rc;
2340	}
2341
2342
2343	/**
2344	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2345	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2346	* the host after every successful VM exit.
2347	*
2348	* @returns VBox status code.
2349	* @param pVM Pointer to the VM.
2350	* @param pVCpu Pointer to the VMCPU.
2351	*/
2352	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2353	{
2354	AssertPtr(pVCpu);
2355	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2356
2357	int rc = VINF_SUCCESS;
2358	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2359	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2360	uint32_t cHostMsrs = 0;
2361	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2362
2363	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2364	{
2365	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2366
2367	# if HC_ARCH_BITS == 64
2368	/* Paranoia. 64-bit code requires these bits to be set always. */
2369	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2370
2371	/*
2372	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2373	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2374	* some reason (e.g. allow transparent reads) we would activate the code below.
2375	*/
2376	# if 0
2377	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2378	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2379	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2380	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2381	if (CPUMIsGuestInLongMode(pVCpu))
2382	{
2383	uint64_t u64GuestEfer;
2384	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2385	AssertRC(rc);
2386
2387	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2388	{
2389	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2390	pHostMsr->u32Reserved = 0;
2391	pHostMsr->u64Value = u64HostEfer;
2392	pHostMsr++; cHostMsrs++;
2393	}
2394	}
2395	# endif
2396	# else /* HC_ARCH_BITS != 64 */
2397	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2398	pHostMsr->u32Reserved = 0;
2399	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2400	if (CPUMIsGuestInLongMode(pVCpu))
2401	{
2402	/* Must match the EFER value in our 64 bits switcher. */
2403	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2404	}
2405	else
2406	# endif
2407	pHostMsr->u64Value = u64HostEfer;
2408	pHostMsr++; cHostMsrs++;
2409	# endif /* HC_ARCH_BITS == 64 */
2410	}
2411
2412	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2413	if (HMVMX_IS_64BIT_HOST_MODE())
2414	{
2415	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2416	pHostMsr->u32Reserved = 0;
2417	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2418	pHostMsr++; cHostMsrs++;
2419	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2420	pHostMsr->u32Reserved = 0;
2421	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2422	pHostMsr++; cHostMsrs++;
2423	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2424	pHostMsr->u32Reserved = 0;
2425	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2426	pHostMsr++; cHostMsrs++;
2427	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2428	pHostMsr->u32Reserved = 0;
2429	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2430	pHostMsr++; cHostMsrs++;
2431	}
2432	# endif
2433
2434	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2435	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2436	{
2437	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2438	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2439	}
2440
2441	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2442	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2443
2444	/*
2445	* Host Sysenter MSRs.
2446	*/
2447	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2448	AssertRCReturn(rc, rc);
2449	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2450	if (HMVMX_IS_64BIT_HOST_MODE())
2451	{
2452	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2453	AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2455	}
2456	else
2457	{
2458	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2459	AssertRCReturn(rc, rc);
2460	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2461	}
2462	#elif HC_ARCH_BITS == 32
2463	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2464	AssertRCReturn(rc, rc);
2465	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2466	#else
2467	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2468	AssertRCReturn(rc, rc);
2469	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2470	#endif
2471	AssertRCReturn(rc, rc);
2472
2473	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2474	* hmR0VmxSetupExitCtls() !! */
2475	return rc;
2476	}
2477
2478
2479	/**
2480	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2481	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2482	* controls".
2483	*
2484	* @returns VBox status code.
2485	* @param pVCpu Pointer to the VMCPU.
2486	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2487	* out-of-sync. Make sure to update the required fields
2488	* before using them.
2489	*
2490	* @remarks No-long-jump zone!!!
2491	*/
2492	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2493	{
2494	int rc = VINF_SUCCESS;
2495	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2496	{
2497	PVM pVM = pVCpu->CTX_SUFF(pVM);
2498	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2499	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2500
2501	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2502	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2503
2504	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2505	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2506	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2507	else
2508	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2509
2510	/*
2511	* The following should not be set (since we're not in SMM mode):
2512	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2513	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2514	*/
2515
2516	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2517	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2518	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2519
2520	if ((val & zap) != val)
2521	{
2522	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2523	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2525	}
2526
2527	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2528	AssertRCReturn(rc, rc);
2529
2530	/* Update VCPU with the currently set VM-exit controls. */
2531	pVCpu->hm.s.vmx.u32EntryCtls = val;
2532	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2533	}
2534	return rc;
2535	}
2536
2537
2538	/**
2539	* Sets up the VM-exit controls in the VMCS.
2540	*
2541	* @returns VBox status code.
2542	* @param pVM Pointer to the VM.
2543	* @param pVCpu Pointer to the VMCPU.
2544	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2545	* out-of-sync. Make sure to update the required fields
2546	* before using them.
2547	*
2548	* @remarks requires EFER.
2549	*/
2550	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2551	{
2552	int rc = VINF_SUCCESS;
2553	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2554	{
2555	PVM pVM = pVCpu->CTX_SUFF(pVM);
2556	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2557	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2558
2559	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2560	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2561
2562	/*
2563	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2564	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2565	*/
2566	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2567	if (HMVMX_IS_64BIT_HOST_MODE())
2568	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2569	else
2570	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2571	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2572	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2573	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2574	else
2575	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2576	#endif
2577
2578	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2579	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2580
2581	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2582	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2583	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2584	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2585	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2586
2587	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2588	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2589
2590	if ((val & zap) != val)
2591	{
2592	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2593	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2594	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2595	}
2596
2597	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2598	AssertRCReturn(rc, rc);
2599
2600	/* Update VCPU with the currently set VM-exit controls. */
2601	pVCpu->hm.s.vmx.u32ExitCtls = val;
2602	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2603	}
2604	return rc;
2605	}
2606
2607
2608	/**
2609	* Loads the guest APIC and related state.
2610	*
2611	* @returns VBox status code.
2612	* @param pVM Pointer to the VM.
2613	* @param pVCpu Pointer to the VMCPU.
2614	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2615	* out-of-sync. Make sure to update the required fields
2616	* before using them.
2617	*/
2618	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2619	{
2620	int rc = VINF_SUCCESS;
2621	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2622	{
2623	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2624	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2625	{
2626	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2627
2628	bool fPendingIntr = false;
2629	uint8_t u8Tpr = 0;
2630	uint8_t u8PendingIntr = 0;
2631	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2632	AssertRCReturn(rc, rc);
2633
2634	/*
2635	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2636	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2637	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2638	* the interrupt when we VM-exit for other reasons.
2639	*/
2640	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2641	uint32_t u32TprThreshold = 0;
2642	if (fPendingIntr)
2643	{
2644	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2645	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2646	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2647	if (u8PendingPriority <= u8TprPriority)
2648	u32TprThreshold = u8PendingPriority;
2649	else
2650	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2651	}
2652	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2653
2654	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2655	AssertRCReturn(rc, rc);
2656	}
2657
2658	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2659	}
2660	return rc;
2661	}
2662
2663
2664	/**
2665	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2666	*
2667	* @returns Guest's interruptibility-state.
2668	* @param pVCpu Pointer to the VMCPU.
2669	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2670	* out-of-sync. Make sure to update the required fields
2671	* before using them.
2672	*
2673	* @remarks No-long-jump zone!!!
2674	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2675	*/
2676	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2677	{
2678	/*
2679	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2680	* inhibit interrupts or clear any existing interrupt-inhibition.
2681	*/
2682	uint32_t uIntrState = 0;
2683	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2684	{
2685	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2686	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2687	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2688	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2689	{
2690	/*
2691	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2692	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2693	*/
2694	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2695	}
2696	else if (pMixedCtx->eflags.Bits.u1IF)
2697	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2698	else
2699	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2700	}
2701	return uIntrState;
2702	}
2703
2704
2705	/**
2706	* Loads the guest's interruptibility-state into the guest-state area in the
2707	* VMCS.
2708	*
2709	* @returns VBox status code.
2710	* @param pVCpu Pointer to the VMCPU.
2711	* @param uIntrState The interruptibility-state to set.
2712	*/
2713	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2714	{
2715	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2716	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2717	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2718	AssertRCReturn(rc, rc);
2719	return rc;
2720	}
2721
2722
2723	/**
2724	* Loads the guest's RIP into the guest-state area in the VMCS.
2725	*
2726	* @returns VBox status code.
2727	* @param pVCpu Pointer to the VMCPU.
2728	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2729	* out-of-sync. Make sure to update the required fields
2730	* before using them.
2731	*
2732	* @remarks No-long-jump zone!!!
2733	*/
2734	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2735	{
2736	int rc = VINF_SUCCESS;
2737	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2738	{
2739	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2740	AssertRCReturn(rc, rc);
2741	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2742	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2743	}
2744	return rc;
2745	}
2746
2747
2748	/**
2749	* Loads the guest's RSP into the guest-state area in the VMCS.
2750	*
2751	* @returns VBox status code.
2752	* @param pVCpu Pointer to the VMCPU.
2753	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2754	* out-of-sync. Make sure to update the required fields
2755	* before using them.
2756	*
2757	* @remarks No-long-jump zone!!!
2758	*/
2759	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2760	{
2761	int rc = VINF_SUCCESS;
2762	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2763	{
2764	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2765	AssertRCReturn(rc, rc);
2766	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2767	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2768	}
2769	return rc;
2770	}
2771
2772
2773	/**
2774	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2775	*
2776	* @returns VBox status code.
2777	* @param pVCpu Pointer to the VMCPU.
2778	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2779	* out-of-sync. Make sure to update the required fields
2780	* before using them.
2781	*
2782	* @remarks No-long-jump zone!!!
2783	*/
2784	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2785	{
2786	int rc = VINF_SUCCESS;
2787	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2788	{
2789	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2790	Let us assert it as such and use 32-bit VMWRITE. */
2791	Assert(!(pMixedCtx->rflags.u64 >> 32));
2792	X86EFLAGS uEFlags = pMixedCtx->eflags;
2793	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2794	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2795
2796	/*
2797	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2798	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2799	*/
2800	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2801	{
2802	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2803	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2804	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2805	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2806	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2807	}
2808
2809	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2810	AssertRCReturn(rc, rc);
2811
2812	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2813	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2814	}
2815	return rc;
2816	}
2817
2818
2819	/**
2820	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2821	*
2822	* @returns VBox status code.
2823	* @param pVCpu Pointer to the VMCPU.
2824	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2825	* out-of-sync. Make sure to update the required fields
2826	* before using them.
2827	*
2828	* @remarks No-long-jump zone!!!
2829	*/
2830	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2831	{
2832	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2833	AssertRCReturn(rc, rc);
2834	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2835	AssertRCReturn(rc, rc);
2836	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2837	AssertRCReturn(rc, rc);
2838	return rc;
2839	}
2840
2841
2842	/**
2843	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2844	* in the VMCS.
2845	*
2846	* @returns VBox status code.
2847	* @param pVM Pointer to the VM.
2848	* @param pVCpu Pointer to the VMCPU.
2849	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2850	* out-of-sync. Make sure to update the required fields
2851	* before using them.
2852	*
2853	* @remarks No-long-jump zone!!!
2854	*/
2855	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2856	{
2857	int rc = VINF_SUCCESS;
2858	PVM pVM = pVCpu->CTX_SUFF(pVM);
2859
2860	/*
2861	* Guest CR0.
2862	* Guest FPU.
2863	*/
2864	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2865	{
2866	Assert(!(pCtx->cr0 >> 32));
2867	uint32_t u32GuestCR0 = pCtx->cr0;
2868
2869	/* The guest's view (read access) of its CR0 is unblemished. */
2870	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2871	AssertRCReturn(rc, rc);
2872	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2873
2874	/* Setup VT-x's view of the guest CR0. */
2875	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2876	if (pVM->hm.s.fNestedPaging)
2877	{
2878	if (CPUMIsGuestPagingEnabledEx(pCtx))
2879	{
2880	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2881	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2882	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2883	}
2884	else
2885	{
2886	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2887	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2888	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2889	}
2890
2891	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2892	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2893	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2894
2895	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2896	AssertRCReturn(rc, rc);
2897	}
2898	else
2899	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2900
2901	/*
2902	* Guest FPU bits.
2903	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2904	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2905	*/
2906	u32GuestCR0 \|= X86_CR0_NE;
2907	bool fInterceptNM = false;
2908	if (CPUMIsGuestFPUStateActive(pVCpu))
2909	{
2910	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2911	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2912	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2913	}
2914	else
2915	{
2916	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2917	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2918	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2919	}
2920
2921	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2922	bool fInterceptMF = false;
2923	if (!(pCtx->cr0 & X86_CR0_NE))
2924	fInterceptMF = true;
2925
2926	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2927	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2928	{
2929	Assert(PDMVmmDevHeapIsEnabled(pVM));
2930	Assert(pVM->hm.s.vmx.pRealModeTSS);
2931	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2932	fInterceptNM = true;
2933	fInterceptMF = true;
2934	}
2935	else
2936	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2937
2938	if (fInterceptNM)
2939	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2940	else
2941	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2942
2943	if (fInterceptMF)
2944	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2945	else
2946	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2947
2948	/* Additional intercepts for debugging, define these yourself explicitly. */
2949	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2950	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2951	\| RT_BIT(X86_XCPT_DB)
2952	\| RT_BIT(X86_XCPT_DE)
2953	\| RT_BIT(X86_XCPT_NM)
2954	\| RT_BIT(X86_XCPT_UD)
2955	\| RT_BIT(X86_XCPT_NP)
2956	\| RT_BIT(X86_XCPT_SS)
2957	\| RT_BIT(X86_XCPT_GP)
2958	\| RT_BIT(X86_XCPT_PF)
2959	\| RT_BIT(X86_XCPT_MF);
2960	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2961	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2962	#endif
2963
2964	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2965
2966	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2967	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2968	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2969	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2970	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2971	else
2972	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2973
2974	u32GuestCR0 \|= uSetCR0;
2975	u32GuestCR0 &= uZapCR0;
2976	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2977
2978	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2979	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2980	AssertRCReturn(rc, rc);
2981	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2982	AssertRCReturn(rc, rc);
2983	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2984
2985	/*
2986	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2987	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2988	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2989	*/
2990	uint32_t u32CR0Mask = 0;
2991	u32CR0Mask = X86_CR0_PE
2992	\| X86_CR0_NE
2993	\| X86_CR0_WP
2994	\| X86_CR0_PG
2995	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2996	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2997	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2998	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2999	u32CR0Mask &= ~X86_CR0_PE;
3000	if (pVM->hm.s.fNestedPaging)
3001	u32CR0Mask &= ~X86_CR0_WP;
3002
3003	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3004	if (fInterceptNM)
3005	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3006	else
3007	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3008
3009	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3010	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3011	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3012	AssertRCReturn(rc, rc);
3013
3014	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3015	}
3016
3017	/*
3018	* Guest CR2.
3019	* It's always loaded in the assembler code. Nothing to do here.
3020	*/
3021
3022	/*
3023	* Guest CR3.
3024	*/
3025	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3026	{
3027	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3028	if (pVM->hm.s.fNestedPaging)
3029	{
3030	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3031
3032	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3033	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3034	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3035	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3036
3037	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3038	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3039	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3040
3041	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3042	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3043	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3044	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3045
3046	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3047	AssertRCReturn(rc, rc);
3048	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3049
3050	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3051	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3052	{
3053	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3054	if (CPUMIsGuestInPAEModeEx(pCtx))
3055	{
3056	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3057	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3058	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3059	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3060	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3061	}
3062
3063	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3064	have Unrestricted Execution to handle the guest when it's not using paging. */
3065	GCPhysGuestCR3 = pCtx->cr3;
3066	}
3067	else
3068	{
3069	/*
3070	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3071	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3072	* EPT takes care of translating it to host-physical addresses.
3073	*/
3074	RTGCPHYS GCPhys;
3075	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3076	Assert(PDMVmmDevHeapIsEnabled(pVM));
3077
3078	/* We obtain it here every time as the guest could have relocated this PCI region. */
3079	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3080	AssertRCReturn(rc, rc);
3081
3082	GCPhysGuestCR3 = GCPhys;
3083	}
3084
3085	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3086	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3087	}
3088	else
3089	{
3090	/* Non-nested paging case, just use the hypervisor's CR3. */
3091	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3092
3093	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3094	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3095	}
3096	AssertRCReturn(rc, rc);
3097
3098	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3099	}
3100
3101	/*
3102	* Guest CR4.
3103	*/
3104	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3105	{
3106	Assert(!(pCtx->cr4 >> 32));
3107	uint32_t u32GuestCR4 = pCtx->cr4;
3108
3109	/* The guest's view of its CR4 is unblemished. */
3110	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3111	AssertRCReturn(rc, rc);
3112	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3113
3114	/* Setup VT-x's view of the guest CR4. */
3115	/*
3116	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3117	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3118	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3119	*/
3120	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3121	{
3122	Assert(pVM->hm.s.vmx.pRealModeTSS);
3123	Assert(PDMVmmDevHeapIsEnabled(pVM));
3124	u32GuestCR4 &= ~X86_CR4_VME;
3125	}
3126
3127	if (pVM->hm.s.fNestedPaging)
3128	{
3129	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3130	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3131	{
3132	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3133	u32GuestCR4 \|= X86_CR4_PSE;
3134	/* Our identity mapping is a 32 bits page directory. */
3135	u32GuestCR4 &= ~X86_CR4_PAE;
3136	}
3137	/* else use guest CR4.*/
3138	}
3139	else
3140	{
3141	/*
3142	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3143	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3144	*/
3145	switch (pVCpu->hm.s.enmShadowMode)
3146	{
3147	case PGMMODE_REAL: /* Real-mode. */
3148	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3149	case PGMMODE_32_BIT: /* 32-bit paging. */
3150	{
3151	u32GuestCR4 &= ~X86_CR4_PAE;
3152	break;
3153	}
3154
3155	case PGMMODE_PAE: /* PAE paging. */
3156	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3157	{
3158	u32GuestCR4 \|= X86_CR4_PAE;
3159	break;
3160	}
3161
3162	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3163	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3164	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3165	break;
3166	#endif
3167	default:
3168	AssertFailed();
3169	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3170	}
3171	}
3172
3173	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3174	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3175	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3176	u32GuestCR4 \|= uSetCR4;
3177	u32GuestCR4 &= uZapCR4;
3178
3179	/* Write VT-x's view of the guest CR4 into the VMCS. */
3180	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3181	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3182	AssertRCReturn(rc, rc);
3183
3184	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3185	uint32_t u32CR4Mask = 0;
3186	u32CR4Mask = X86_CR4_VME
3187	\| X86_CR4_PAE
3188	\| X86_CR4_PGE
3189	\| X86_CR4_PSE
3190	\| X86_CR4_VMXE;
3191	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3192	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3193	AssertRCReturn(rc, rc);
3194
3195	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3196	}
3197	return rc;
3198	}
3199
3200
3201	/**
3202	* Loads the guest debug registers into the guest-state area in the VMCS.
3203	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3204	*
3205	* @returns VBox status code.
3206	* @param pVCpu Pointer to the VMCPU.
3207	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3208	* out-of-sync. Make sure to update the required fields
3209	* before using them.
3210	*
3211	* @remarks No-long-jump zone!!!
3212	*/
3213	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3214	{
3215	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3216	return VINF_SUCCESS;
3217
3218	#ifdef VBOX_STRICT
3219	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3220	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3221	{
3222	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3223	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3224	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3225	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3226	}
3227	#endif
3228
3229	int rc = VERR_INTERNAL_ERROR_5;
3230	PVM pVM = pVCpu->CTX_SUFF(pVM);
3231	bool fInterceptDB = false;
3232	bool fInterceptMovDRx = false;
3233	if (DBGFIsStepping(pVCpu))
3234	{
3235	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3236	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3237	{
3238	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3239	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3240	AssertRCReturn(rc, rc);
3241	Assert(fInterceptDB == false);
3242	}
3243	else
3244	{
3245	fInterceptDB = true;
3246	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3247	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3248	}
3249	}
3250
3251	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3252	{
3253	if (!CPUMIsHyperDebugStateActive(pVCpu))
3254	{
3255	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3256	AssertRC(rc);
3257	}
3258	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3259	fInterceptMovDRx = true;
3260	}
3261	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3262	{
3263	if (!CPUMIsGuestDebugStateActive(pVCpu))
3264	{
3265	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3266	AssertRC(rc);
3267	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3268	}
3269	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3270	Assert(fInterceptMovDRx == false);
3271	}
3272	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3273	{
3274	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3275	fInterceptMovDRx = true;
3276	}
3277
3278	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3279	if (fInterceptDB)
3280	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3281	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3282	{
3283	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3284	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3285	#endif
3286	}
3287
3288	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3289	if (fInterceptMovDRx)
3290	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3291	else
3292	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3293
3294	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3295	AssertRCReturn(rc, rc);
3296	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3297	AssertRCReturn(rc, rc);
3298
3299	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3300	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3301	AssertRCReturn(rc, rc);
3302
3303	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3304	return rc;
3305	}
3306
3307
3308	#ifdef VBOX_STRICT
3309	/**
3310	* Strict function to validate segment registers.
3311	*
3312	* @remarks Requires CR0.
3313	*/
3314	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3315	{
3316	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3317	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3318	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3319
3320	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3321	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3322	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3323	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3324	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3325	{
3326	/* Protected mode checks */
3327	/* CS */
3328	Assert(pCtx->cs.Attr.n.u1Present);
3329	Assert(!(pCtx->cs.Attr.u & 0xf00));
3330	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3331	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3332	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3333	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3334	\|\| (pCtx->cs.Attr.n.u1Granularity));
3335	/* CS cannot be loaded with NULL in protected mode. */
3336	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3337	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3338	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3339	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3340	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3341	else
3342	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3343	/* SS */
3344	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3345	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3346	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3347	if ( !(pCtx->cr0 & X86_CR0_PE)
3348	\|\| pCtx->cs.Attr.n.u4Type == 3)
3349	{
3350	Assert(!pCtx->ss.Attr.n.u2Dpl);
3351	}
3352	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3353	{
3354	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3355	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3356	Assert(pCtx->ss.Attr.n.u1Present);
3357	Assert(!(pCtx->ss.Attr.u & 0xf00));
3358	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3359	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3360	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3361	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3362	\|\| (pCtx->ss.Attr.n.u1Granularity));
3363	}
3364	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3365	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3366	{
3367	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3368	Assert(pCtx->ds.Attr.n.u1Present);
3369	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3370	Assert(!(pCtx->ds.Attr.u & 0xf00));
3371	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3372	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3373	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3374	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3375	\|\| (pCtx->ds.Attr.n.u1Granularity));
3376	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3377	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3378	}
3379	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3380	{
3381	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3382	Assert(pCtx->es.Attr.n.u1Present);
3383	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3384	Assert(!(pCtx->es.Attr.u & 0xf00));
3385	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3386	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3387	\|\| !(pCtx->es.Attr.n.u1Granularity));
3388	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3389	\|\| (pCtx->es.Attr.n.u1Granularity));
3390	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3391	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3392	}
3393	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3394	{
3395	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3396	Assert(pCtx->fs.Attr.n.u1Present);
3397	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3398	Assert(!(pCtx->fs.Attr.u & 0xf00));
3399	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3400	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3401	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3402	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3403	\|\| (pCtx->fs.Attr.n.u1Granularity));
3404	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3405	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3406	}
3407	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3408	{
3409	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3410	Assert(pCtx->gs.Attr.n.u1Present);
3411	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3412	Assert(!(pCtx->gs.Attr.u & 0xf00));
3413	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3414	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3415	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3416	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3417	\|\| (pCtx->gs.Attr.n.u1Granularity));
3418	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3419	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3420	}
3421	/* 64-bit capable CPUs. */
3422	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3423	Assert(!(pCtx->cs.u64Base >> 32));
3424	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3425	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3426	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3427	# endif
3428	}
3429	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3430	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3431	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3432	{
3433	/* Real and v86 mode checks. */
3434	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3435	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3436	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3437	{
3438	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3439	}
3440	else
3441	{
3442	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3443	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3444	}
3445
3446	/* CS */
3447	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3448	Assert(pCtx->cs.u32Limit == 0xffff);
3449	Assert(u32CSAttr == 0xf3);
3450	/* SS */
3451	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3452	Assert(pCtx->ss.u32Limit == 0xffff);
3453	Assert(u32SSAttr == 0xf3);
3454	/* DS */
3455	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3456	Assert(pCtx->ds.u32Limit == 0xffff);
3457	Assert(u32DSAttr == 0xf3);
3458	/* ES */
3459	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3460	Assert(pCtx->es.u32Limit == 0xffff);
3461	Assert(u32ESAttr == 0xf3);
3462	/* FS */
3463	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3464	Assert(pCtx->fs.u32Limit == 0xffff);
3465	Assert(u32FSAttr == 0xf3);
3466	/* GS */
3467	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3468	Assert(pCtx->gs.u32Limit == 0xffff);
3469	Assert(u32GSAttr == 0xf3);
3470	/* 64-bit capable CPUs. */
3471	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3472	Assert(!(pCtx->cs.u64Base >> 32));
3473	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3474	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3475	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3476	# endif
3477	}
3478	}
3479	#endif /* VBOX_STRICT */
3480
3481
3482	/**
3483	* Writes a guest segment register into the guest-state area in the VMCS.
3484	*
3485	* @returns VBox status code.
3486	* @param pVCpu Pointer to the VMCPU.
3487	* @param idxSel Index of the selector in the VMCS.
3488	* @param idxLimit Index of the segment limit in the VMCS.
3489	* @param idxBase Index of the segment base in the VMCS.
3490	* @param idxAccess Index of the access rights of the segment in the VMCS.
3491	* @param pSelReg Pointer to the segment selector.
3492	* @param pCtx Pointer to the guest-CPU context.
3493	*
3494	* @remarks No-long-jump zone!!!
3495	*/
3496	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3497	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3498	{
3499	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3500	AssertRCReturn(rc, rc);
3501	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3502	AssertRCReturn(rc, rc);
3503	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3504	AssertRCReturn(rc, rc);
3505
3506	uint32_t u32Access = pSelReg->Attr.u;
3507	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3508	{
3509	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3510	u32Access = 0xf3;
3511	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3512	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3513	}
3514	else
3515	{
3516	/*
3517	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3518	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3519	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3520	* loaded in protected-mode have their attribute as 0.
3521	*/
3522	if (!u32Access)
3523	u32Access = HMVMX_SEL_UNUSABLE;
3524	}
3525
3526	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3527	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3528	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3529
3530	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3531	AssertRCReturn(rc, rc);
3532	return rc;
3533	}
3534
3535
3536	/**
3537	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3538	* into the guest-state area in the VMCS.
3539	*
3540	* @returns VBox status code.
3541	* @param pVM Pointer to the VM.
3542	* @param pVCPU Pointer to the VMCPU.
3543	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3544	* out-of-sync. Make sure to update the required fields
3545	* before using them.
3546	*
3547	* @remarks Requires CR0 (strict builds validation).
3548	* @remarks No-long-jump zone!!!
3549	*/
3550	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3551	{
3552	int rc = VERR_INTERNAL_ERROR_5;
3553	PVM pVM = pVCpu->CTX_SUFF(pVM);
3554
3555	/*
3556	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3557	*/
3558	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3559	{
3560	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3561	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3562	{
3563	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3564	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3565	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3566	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3567	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3568	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3569	}
3570
3571	#ifdef VBOX_WITH_REM
3572	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3573	{
3574	Assert(pVM->hm.s.vmx.pRealModeTSS);
3575	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3576	if ( pVCpu->hm.s.vmx.fWasInRealMode
3577	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3578	{
3579	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3580	in real-mode (e.g. OpenBSD 4.0) */
3581	REMFlushTBs(pVM);
3582	Log4(("Load: Switch to protected mode detected!\n"));
3583	pVCpu->hm.s.vmx.fWasInRealMode = false;
3584	}
3585	}
3586	#endif
3587	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3588	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3589	AssertRCReturn(rc, rc);
3590	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3591	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3592	AssertRCReturn(rc, rc);
3593	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3594	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3595	AssertRCReturn(rc, rc);
3596	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3597	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3598	AssertRCReturn(rc, rc);
3599	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3600	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3601	AssertRCReturn(rc, rc);
3602	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3603	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3604	AssertRCReturn(rc, rc);
3605
3606	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3607	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3608	#ifdef VBOX_STRICT
3609	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3610	#endif
3611	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3612	}
3613
3614	/*
3615	* Guest TR.
3616	*/
3617	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3618	{
3619	/*
3620	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3621	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3622	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3623	*/
3624	uint16_t u16Sel = 0;
3625	uint32_t u32Limit = 0;
3626	uint64_t u64Base = 0;
3627	uint32_t u32AccessRights = 0;
3628
3629	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3630	{
3631	u16Sel = pMixedCtx->tr.Sel;
3632	u32Limit = pMixedCtx->tr.u32Limit;
3633	u64Base = pMixedCtx->tr.u64Base;
3634	u32AccessRights = pMixedCtx->tr.Attr.u;
3635	}
3636	else
3637	{
3638	Assert(pVM->hm.s.vmx.pRealModeTSS);
3639	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3640
3641	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3642	RTGCPHYS GCPhys;
3643	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3644	AssertRCReturn(rc, rc);
3645
3646	X86DESCATTR DescAttr;
3647	DescAttr.u = 0;
3648	DescAttr.n.u1Present = 1;
3649	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3650
3651	u16Sel = 0;
3652	u32Limit = HM_VTX_TSS_SIZE;
3653	u64Base = GCPhys; /* in real-mode phys = virt. */
3654	u32AccessRights = DescAttr.u;
3655	}
3656
3657	/* Validate. */
3658	Assert(!(u16Sel & RT_BIT(2)));
3659	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3660	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3661	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3662	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3663	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3664	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3665	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3666	Assert( (u32Limit & 0xfff) == 0xfff
3667	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3668	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3669	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3670
3671	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3672	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3673	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3674	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3675
3676	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3677	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3678	}
3679
3680	/*
3681	* Guest GDTR.
3682	*/
3683	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3684	{
3685	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3686	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3687
3688	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3689	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3690	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3691	}
3692
3693	/*
3694	* Guest LDTR.
3695	*/
3696	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3697	{
3698	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3699	uint32_t u32Access = 0;
3700	if (!pMixedCtx->ldtr.Attr.u)
3701	u32Access = HMVMX_SEL_UNUSABLE;
3702	else
3703	u32Access = pMixedCtx->ldtr.Attr.u;
3704
3705	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3706	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3707	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3708	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3709
3710	/* Validate. */
3711	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3712	{
3713	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3714	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3715	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3716	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3717	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3718	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3719	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3720	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3721	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3722	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3723	}
3724
3725	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3726	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3727	}
3728
3729	/*
3730	* Guest IDTR.
3731	*/
3732	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3733	{
3734	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3735	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3736
3737	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3738	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3739	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3740	}
3741
3742	return VINF_SUCCESS;
3743	}
3744
3745
3746	/**
3747	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3748	* areas. These MSRs will automatically be loaded to the host CPU on every
3749	* successful VM entry and stored from the host CPU on every successful VM exit.
3750	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3751	*
3752	* @returns VBox status code.
3753	* @param pVCpu Pointer to the VMCPU.
3754	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3755	* out-of-sync. Make sure to update the required fields
3756	* before using them.
3757	*
3758	* @remarks No-long-jump zone!!!
3759	*/
3760	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3761	{
3762	AssertPtr(pVCpu);
3763	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3764
3765	/*
3766	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3767	*/
3768	int rc = VINF_SUCCESS;
3769	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3770	{
3771	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3772	PVM pVM = pVCpu->CTX_SUFF(pVM);
3773	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3774	uint32_t cGuestMsrs = 0;
3775
3776	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3777	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3778	* when the guest really is in 64-bit mode. */
3779	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3780	if (fSupportsLongMode)
3781	{
3782	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3783	pGuestMsr->u32Reserved = 0;
3784	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3785	pGuestMsr++; cGuestMsrs++;
3786	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3787	pGuestMsr->u32Reserved = 0;
3788	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3789	pGuestMsr++; cGuestMsrs++;
3790	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3791	pGuestMsr->u32Reserved = 0;
3792	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3793	pGuestMsr++; cGuestMsrs++;
3794	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3795	pGuestMsr->u32Reserved = 0;
3796	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3797	pGuestMsr++; cGuestMsrs++;
3798	}
3799
3800	/*
3801	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3802	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3803	*/
3804	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3805	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3806	{
3807	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3808	pGuestMsr->u32Reserved = 0;
3809	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3810	AssertRCReturn(rc, rc);
3811	pGuestMsr++; cGuestMsrs++;
3812	}
3813
3814	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3815	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3816	{
3817	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3818	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3819	}
3820
3821	/* Update the VCPU's copy of the guest MSR count. */
3822	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3823	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3824	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3825	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3826
3827	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3828	}
3829
3830	/*
3831	* Guest Sysenter MSRs.
3832	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3833	* VM-exits on WRMSRs for these MSRs.
3834	*/
3835	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3836	{
3837	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3838	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3839	}
3840	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3841	{
3842	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3843	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3844	}
3845	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3846	{
3847	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3848	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3849	}
3850
3851	return rc;
3852	}
3853
3854
3855	/**
3856	* Loads the guest activity state into the guest-state area in the VMCS.
3857	*
3858	* @returns VBox status code.
3859	* @param pVCpu Pointer to the VMCPU.
3860	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3861	* out-of-sync. Make sure to update the required fields
3862	* before using them.
3863	*
3864	* @remarks No-long-jump zone!!!
3865	*/
3866	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3867	{
3868	/** @todo See if we can make use of other states, e.g.
3869	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3870	int rc = VINF_SUCCESS;
3871	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3872	{
3873	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3874	AssertRCReturn(rc, rc);
3875	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3876	}
3877	return rc;
3878	}
3879
3880
3881	/**
3882	* Sets up the appropriate function to run guest code.
3883	*
3884	* @returns VBox status code.
3885	* @param pVCpu Pointer to the VMCPU.
3886	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3887	* out-of-sync. Make sure to update the required fields
3888	* before using them.
3889	*
3890	* @remarks No-long-jump zone!!!
3891	*/
3892	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3893	{
3894	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3895	{
3896	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3897	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3898	#endif
3899	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3900	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3901	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3902	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3903	#else
3904	/* 64-bit host or hybrid host. */
3905	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3906	#endif
3907	}
3908	else
3909	{
3910	/* Guest is not in long mode, use the 32-bit handler. */
3911	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3912	}
3913	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3914	return VINF_SUCCESS;
3915	}
3916
3917
3918	/**
3919	* Wrapper for running the guest code in VT-x.
3920	*
3921	* @returns VBox strict status code.
3922	* @param pVM Pointer to the VM.
3923	* @param pVCpu Pointer to the VMCPU.
3924	* @param pCtx Pointer to the guest-CPU context.
3925	*
3926	* @remarks No-long-jump zone!!!
3927	*/
3928	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3929	{
3930	/*
3931	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3932	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3933	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3934	*/
3935	#ifdef VBOX_WITH_KERNEL_USING_XMM
3936	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3937	#else
3938	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3939	#endif
3940	}
3941
3942
3943	/**
3944	* Reports world-switch error and dumps some useful debug info.
3945	*
3946	* @param pVM Pointer to the VM.
3947	* @param pVCpu Pointer to the VMCPU.
3948	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3949	* @param pCtx Pointer to the guest-CPU context.
3950	* @param pVmxTransient Pointer to the VMX transient structure (only
3951	* exitReason updated).
3952	*/
3953	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3954	{
3955	Assert(pVM);
3956	Assert(pVCpu);
3957	Assert(pCtx);
3958	Assert(pVmxTransient);
3959	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3960
3961	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3962	switch (rcVMRun)
3963	{
3964	case VERR_VMX_INVALID_VMXON_PTR:
3965	AssertFailed();
3966	break;
3967	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3968	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3969	{
3970	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3971	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3972	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3973	AssertRC(rc);
3974
3975	#ifdef VBOX_STRICT
3976	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3977	pVmxTransient->uExitReason));
3978	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3979	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3980	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3981	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3982	else
3983	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3984
3985	/* VMX control bits. */
3986	uint32_t u32Val;
3987	uint64_t u64Val;
3988	HMVMXHCUINTREG uHCReg;
3989	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3990	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3991	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3992	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3993	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3994	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3995	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3996	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3997	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3998	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3999	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4000	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4001	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4002	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4003	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4004	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4005	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4006	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4007	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4008	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4009	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4010	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4011	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4012	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4013	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4014	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4015	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4016	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4017	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4018	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4019	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4020	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4021	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4022	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4023	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4024	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4025	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4026	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4027	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4028	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4029	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4030	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4031
4032	/* Guest bits. */
4033	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4034	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4035	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4036	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4037	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4038	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4039	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4040	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4041
4042	/* Host bits. */
4043	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4044	Log4(("Host CR0 %#RHr\n", uHCReg));
4045	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4046	Log4(("Host CR3 %#RHr\n", uHCReg));
4047	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4048	Log4(("Host CR4 %#RHr\n", uHCReg));
4049
4050	RTGDTR HostGdtr;
4051	PCX86DESCHC pDesc;
4052	ASMGetGDTR(&HostGdtr);
4053	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4054	Log4(("Host CS %#08x\n", u32Val));
4055	if (u32Val < HostGdtr.cbGdt)
4056	{
4057	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4058	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4059	}
4060
4061	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4062	Log4(("Host DS %#08x\n", u32Val));
4063	if (u32Val < HostGdtr.cbGdt)
4064	{
4065	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4066	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4067	}
4068
4069	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4070	Log4(("Host ES %#08x\n", u32Val));
4071	if (u32Val < HostGdtr.cbGdt)
4072	{
4073	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4074	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4075	}
4076
4077	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4078	Log4(("Host FS %#08x\n", u32Val));
4079	if (u32Val < HostGdtr.cbGdt)
4080	{
4081	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4082	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4083	}
4084
4085	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4086	Log4(("Host GS %#08x\n", u32Val));
4087	if (u32Val < HostGdtr.cbGdt)
4088	{
4089	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4090	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4091	}
4092
4093	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4094	Log4(("Host SS %#08x\n", u32Val));
4095	if (u32Val < HostGdtr.cbGdt)
4096	{
4097	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4098	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4099	}
4100
4101	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4102	Log4(("Host TR %#08x\n", u32Val));
4103	if (u32Val < HostGdtr.cbGdt)
4104	{
4105	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4106	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4107	}
4108
4109	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4110	Log4(("Host TR Base %#RHv\n", uHCReg));
4111	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4112	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4113	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4114	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4115	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4116	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4117	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4118	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4119	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4120	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4121	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4122	Log4(("Host RSP %#RHv\n", uHCReg));
4123	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4124	Log4(("Host RIP %#RHv\n", uHCReg));
4125	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4126	if (HMVMX_IS_64BIT_HOST_MODE())
4127	{
4128	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4129	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4130	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4131	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4132	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4133	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4134	}
4135	# endif
4136	#endif /* VBOX_STRICT */
4137	break;
4138	}
4139
4140	default:
4141	/* Impossible */
4142	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4143	break;
4144	}
4145	NOREF(pVM);
4146	}
4147
4148
4149	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4150	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4151	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4152	#endif
4153	#ifdef VBOX_STRICT
4154	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4155	{
4156	switch (idxField)
4157	{
4158	case VMX_VMCS_GUEST_RIP:
4159	case VMX_VMCS_GUEST_RSP:
4160	case VMX_VMCS_GUEST_SYSENTER_EIP:
4161	case VMX_VMCS_GUEST_SYSENTER_ESP:
4162	case VMX_VMCS_GUEST_GDTR_BASE:
4163	case VMX_VMCS_GUEST_IDTR_BASE:
4164	case VMX_VMCS_GUEST_CS_BASE:
4165	case VMX_VMCS_GUEST_DS_BASE:
4166	case VMX_VMCS_GUEST_ES_BASE:
4167	case VMX_VMCS_GUEST_FS_BASE:
4168	case VMX_VMCS_GUEST_GS_BASE:
4169	case VMX_VMCS_GUEST_SS_BASE:
4170	case VMX_VMCS_GUEST_LDTR_BASE:
4171	case VMX_VMCS_GUEST_TR_BASE:
4172	case VMX_VMCS_GUEST_CR3:
4173	return true;
4174	}
4175	return false;
4176	}
4177
4178	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4179	{
4180	switch (idxField)
4181	{
4182	/* Read-only fields. */
4183	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4184	return true;
4185	}
4186	/* Remaining readable fields should also be writable. */
4187	return hmR0VmxIsValidWriteField(idxField);
4188	}
4189	#endif /* VBOX_STRICT */
4190
4191
4192	/**
4193	* Executes the specified handler in 64-bit mode.
4194	*
4195	* @returns VBox status code.
4196	* @param pVM Pointer to the VM.
4197	* @param pVCpu Pointer to the VMCPU.
4198	* @param pCtx Pointer to the guest CPU context.
4199	* @param enmOp The operation to perform.
4200	* @param cbParam Number of parameters.
4201	* @param paParam Array of 32-bit parameters.
4202	*/
4203	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4204	uint32_t *paParam)
4205	{
4206	int rc, rc2;
4207	PHMGLOBLCPUINFO pCpu;
4208	RTHCPHYS HCPhysCpuPage;
4209	RTCCUINTREG uOldEFlags;
4210
4211	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4212	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4213	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4214	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4215
4216	#ifdef VBOX_STRICT
4217	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4218	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4219
4220	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4221	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4222	#endif
4223
4224	/* Disable interrupts. */
4225	uOldEFlags = ASMIntDisableFlags();
4226
4227	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4228	RTCPUID idHostCpu = RTMpCpuId();
4229	CPUMR0SetLApic(pVM, idHostCpu);
4230	#endif
4231
4232	pCpu = HMR0GetCurrentCpu();
4233	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4234
4235	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4236	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4237
4238	/* Leave VMX Root Mode. */
4239	VMXDisable();
4240
4241	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4242
4243	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4244	CPUMSetHyperEIP(pVCpu, enmOp);
4245	for (int i = (int)cbParam - 1; i >= 0; i--)
4246	CPUMPushHyper(pVCpu, paParam[i]);
4247
4248	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4249
4250	/* Call the switcher. */
4251	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4252	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4253
4254	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4255	/* Make sure the VMX instructions don't cause #UD faults. */
4256	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4257
4258	/* Re-enter VMX Root Mode */
4259	rc2 = VMXEnable(HCPhysCpuPage);
4260	if (RT_FAILURE(rc2))
4261	{
4262	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4263	ASMSetFlags(uOldEFlags);
4264	return rc2;
4265	}
4266
4267	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4268	AssertRC(rc2);
4269	Assert(!(ASMGetFlags() & X86_EFL_IF));
4270	ASMSetFlags(uOldEFlags);
4271	return rc;
4272	}
4273
4274
4275	/**
4276	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4277	* supporting 64-bit guests.
4278	*
4279	* @returns VBox status code.
4280	* @param fResume Whether to VMLAUNCH or VMRESUME.
4281	* @param pCtx Pointer to the guest-CPU context.
4282	* @param pCache Pointer to the VMCS cache.
4283	* @param pVM Pointer to the VM.
4284	* @param pVCpu Pointer to the VMCPU.
4285	*/
4286	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4287	{
4288	uint32_t aParam[6];
4289	PHMGLOBLCPUINFO pCpu = NULL;
4290	RTHCPHYS HCPhysCpuPage = 0;
4291	int rc = VERR_INTERNAL_ERROR_5;
4292
4293	pCpu = HMR0GetCurrentCpu();
4294	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4295
4296	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4297	pCache->uPos = 1;
4298	pCache->interPD = PGMGetInterPaeCR3(pVM);
4299	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4300	#endif
4301
4302	#ifdef VBOX_STRICT
4303	pCache->TestIn.HCPhysCpuPage = 0;
4304	pCache->TestIn.HCPhysVmcs = 0;
4305	pCache->TestIn.pCache = 0;
4306	pCache->TestOut.HCPhysVmcs = 0;
4307	pCache->TestOut.pCache = 0;
4308	pCache->TestOut.pCtx = 0;
4309	pCache->TestOut.eflags = 0;
4310	#endif
4311
4312	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4313	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4314	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4315	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4316	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4317	aParam[5] = 0;
4318
4319	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4320	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4321	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4322	#endif
4323	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4324
4325	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4326	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4327	Assert(pCtx->dr[4] == 10);
4328	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4329	#endif
4330
4331	#ifdef VBOX_STRICT
4332	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4333	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4334	pVCpu->hm.s.vmx.HCPhysVmcs));
4335	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4336	pCache->TestOut.HCPhysVmcs));
4337	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4338	pCache->TestOut.pCache));
4339	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4340	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4341	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4342	pCache->TestOut.pCtx));
4343	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4344	#endif
4345	return rc;
4346	}
4347
4348
4349	/**
4350	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4351	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4352	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4353	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4354	*
4355	* @returns VBox status code.
4356	* @param pVM Pointer to the VM.
4357	* @param pVCpu Pointer to the VMCPU.
4358	*/
4359	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4360	{
4361	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4362	{ \
4363	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4364	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4365	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4366	++cReadFields; \
4367	}
4368
4369	AssertPtr(pVM);
4370	AssertPtr(pVCpu);
4371	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4372	uint32_t cReadFields = 0;
4373
4374	/*
4375	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4376	* and serve to indicate exceptions to the rules.
4377	*/
4378
4379	/* Guest-natural selector base fields. */
4380	#if 0
4381	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4382	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4383	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4384	#endif
4385	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4387	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4388	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4390	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4392	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4393	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4396	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4397	#if 0
4398	/* Unused natural width guest-state fields. */
4399	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4400	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4401	#endif
4402	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4404
4405	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4406	#if 0
4407	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4408	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4409	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4410	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4412	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4413	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4414	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4415	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4416	#endif
4417
4418	/* Natural width guest-state fields. */
4419	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4420	#if 0
4421	/* Currently unused field. */
4422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4423	#endif
4424
4425	if (pVM->hm.s.fNestedPaging)
4426	{
4427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4428	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4429	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4430	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4431	}
4432	else
4433	{
4434	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4435	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4436	}
4437
4438	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4439	return VINF_SUCCESS;
4440	}
4441
4442
4443	/**
4444	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4445	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4446	* darwin, running 64-bit guests).
4447	*
4448	* @returns VBox status code.
4449	* @param pVCpu Pointer to the VMCPU.
4450	* @param idxField The VMCS field encoding.
4451	* @param u64Val 16, 32 or 64 bits value.
4452	*/
4453	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4454	{
4455	int rc;
4456	switch (idxField)
4457	{
4458	/*
4459	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4460	*/
4461	/* 64-bit Control fields. */
4462	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4463	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4464	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4465	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4466	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4467	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4468	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4469	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4470	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4471	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4472	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4473	case VMX_VMCS64_CTRL_EPTP_FULL:
4474	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4475	/* 64-bit Guest-state fields. */
4476	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4477	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4478	case VMX_VMCS64_GUEST_PAT_FULL:
4479	case VMX_VMCS64_GUEST_EFER_FULL:
4480	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4481	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4482	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4483	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4484	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4485	/* 64-bit Host-state fields. */
4486	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4487	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4488	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4489	{
4490	rc = VMXWriteVmcs32(idxField, u64Val);
4491	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4492	break;
4493	}
4494
4495	/*
4496	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4497	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4498	*/
4499	/* Natural-width Guest-state fields. */
4500	case VMX_VMCS_GUEST_CR3:
4501	case VMX_VMCS_GUEST_ES_BASE:
4502	case VMX_VMCS_GUEST_CS_BASE:
4503	case VMX_VMCS_GUEST_SS_BASE:
4504	case VMX_VMCS_GUEST_DS_BASE:
4505	case VMX_VMCS_GUEST_FS_BASE:
4506	case VMX_VMCS_GUEST_GS_BASE:
4507	case VMX_VMCS_GUEST_LDTR_BASE:
4508	case VMX_VMCS_GUEST_TR_BASE:
4509	case VMX_VMCS_GUEST_GDTR_BASE:
4510	case VMX_VMCS_GUEST_IDTR_BASE:
4511	case VMX_VMCS_GUEST_RSP:
4512	case VMX_VMCS_GUEST_RIP:
4513	case VMX_VMCS_GUEST_SYSENTER_ESP:
4514	case VMX_VMCS_GUEST_SYSENTER_EIP:
4515	{
4516	if (!(u64Val >> 32))
4517	{
4518	/* If this field is 64-bit, VT-x will zero out the top bits. */
4519	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4520	}
4521	else
4522	{
4523	/* Assert that only the 32->64 switcher case should ever come here. */
4524	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4525	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4526	}
4527	break;
4528	}
4529
4530	default:
4531	{
4532	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4533	rc = VERR_INVALID_PARAMETER;
4534	break;
4535	}
4536	}
4537	AssertRCReturn(rc, rc);
4538	return rc;
4539	}
4540
4541
4542	/**
4543	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4544	* hosts (except darwin) for 64-bit guests.
4545	*
4546	* @param pVCpu Pointer to the VMCPU.
4547	* @param idxField The VMCS field encoding.
4548	* @param u64Val 16, 32 or 64 bits value.
4549	*/
4550	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4551	{
4552	AssertPtr(pVCpu);
4553	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4554
4555	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4556	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4557
4558	/* Make sure there are no duplicates. */
4559	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4560	{
4561	if (pCache->Write.aField[i] == idxField)
4562	{
4563	pCache->Write.aFieldVal[i] = u64Val;
4564	return VINF_SUCCESS;
4565	}
4566	}
4567
4568	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4569	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4570	pCache->Write.cValidEntries++;
4571	return VINF_SUCCESS;
4572	}
4573
4574	/* Enable later when the assembly code uses these as callbacks. */
4575	#if 0
4576	/*
4577	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4578	*
4579	* @param pVCpu Pointer to the VMCPU.
4580	* @param pCache Pointer to the VMCS cache.
4581	*
4582	* @remarks No-long-jump zone!!!
4583	*/
4584	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4585	{
4586	AssertPtr(pCache);
4587	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4588	{
4589	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4590	AssertRC(rc);
4591	}
4592	pCache->Write.cValidEntries = 0;
4593	}
4594
4595
4596	/**
4597	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4598	*
4599	* @param pVCpu Pointer to the VMCPU.
4600	* @param pCache Pointer to the VMCS cache.
4601	*
4602	* @remarks No-long-jump zone!!!
4603	*/
4604	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4605	{
4606	AssertPtr(pCache);
4607	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4608	{
4609	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4610	AssertRC(rc);
4611	}
4612	}
4613	#endif
4614	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4615
4616
4617	/**
4618	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4619	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4620	* timer.
4621	*
4622	* @returns VBox status code.
4623	* @param pVCpu Pointer to the VMCPU.
4624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4625	* out-of-sync. Make sure to update the required fields
4626	* before using them.
4627	* @remarks No-long-jump zone!!!
4628	*/
4629	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4630	{
4631	int rc = VERR_INTERNAL_ERROR_5;
4632	bool fOffsettedTsc = false;
4633	PVM pVM = pVCpu->CTX_SUFF(pVM);
4634	if (pVM->hm.s.vmx.fUsePreemptTimer)
4635	{
4636	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4637
4638	/* Make sure the returned values have sane upper and lower boundaries. */
4639	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4640	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4641	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4642	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4643
4644	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4645	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4646	}
4647	else
4648	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4649
4650	if (fOffsettedTsc)
4651	{
4652	uint64_t u64CurTSC = ASMReadTSC();
4653	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4654	{
4655	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4656	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4657
4658	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4659	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4660	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4661	}
4662	else
4663	{
4664	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4665	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4666	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4667	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4668	}
4669	}
4670	else
4671	{
4672	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4673	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4674	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4675	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4676	}
4677	}
4678
4679
4680	/**
4681	* Determines if an exception is a contributory exception. Contributory
4682	* exceptions are ones which can cause double-faults. Page-fault is
4683	* intentionally not included here as it's a conditional contributory exception.
4684	*
4685	* @returns true if the exception is contributory, false otherwise.
4686	* @param uVector The exception vector.
4687	*/
4688	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4689	{
4690	switch (uVector)
4691	{
4692	case X86_XCPT_GP:
4693	case X86_XCPT_SS:
4694	case X86_XCPT_NP:
4695	case X86_XCPT_TS:
4696	case X86_XCPT_DE:
4697	return true;
4698	default:
4699	break;
4700	}
4701	return false;
4702	}
4703
4704
4705	/**
4706	* Sets an event as a pending event to be injected into the guest.
4707	*
4708	* @param pVCpu Pointer to the VMCPU.
4709	* @param u32IntrInfo The VM-entry interruption-information field.
4710	* @param cbInstr The VM-entry instruction length in bytes (for software
4711	* interrupts, exceptions and privileged software
4712	* exceptions).
4713	* @param u32ErrCode The VM-entry exception error code.
4714	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4715	* page-fault.
4716	*/
4717	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4718	RTGCUINTPTR GCPtrFaultAddress)
4719	{
4720	Assert(!pVCpu->hm.s.Event.fPending);
4721	pVCpu->hm.s.Event.fPending = true;
4722	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4723	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4724	pVCpu->hm.s.Event.cbInstr = cbInstr;
4725	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4726	}
4727
4728
4729	/**
4730	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4731	*
4732	* @param pVCpu Pointer to the VMCPU.
4733	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4734	* out-of-sync. Make sure to update the required fields
4735	* before using them.
4736	*/
4737	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4738	{
4739	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4740	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4741	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4742	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4743	}
4744
4745
4746	/**
4747	* Handle a condition that occurred while delivering an event through the guest
4748	* IDT.
4749	*
4750	* @returns VBox status code (informational error codes included).
4751	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4752	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4753	* continue execution of the guest which will delivery the #DF.
4754	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4755	*
4756	* @param pVCpu Pointer to the VMCPU.
4757	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4758	* out-of-sync. Make sure to update the required fields
4759	* before using them.
4760	* @param pVmxTransient Pointer to the VMX transient structure.
4761	*
4762	* @remarks No-long-jump zone!!!
4763	*/
4764	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4765	{
4766	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4767	AssertRC(rc);
4768	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4769	{
4770	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4771	AssertRCReturn(rc, rc);
4772
4773	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4774	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4775	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4776
4777	typedef enum
4778	{
4779	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4780	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4781	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4782	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4783	} VMXREFLECTXCPT;
4784
4785	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4786	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4787	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4788	{
4789	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4790	{
4791	enmReflect = VMXREFLECTXCPT_XCPT;
4792	#ifdef VBOX_STRICT
4793	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4794	&& uExitVector == X86_XCPT_PF)
4795	{
4796	Log4(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4797	}
4798	#endif
4799	if ( uExitVector == X86_XCPT_PF
4800	&& uIdtVector == X86_XCPT_PF)
4801	{
4802	pVmxTransient->fVectoringPF = true;
4803	Log4(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4804	}
4805	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4806	&& hmR0VmxIsContributoryXcpt(uExitVector)
4807	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4808	\|\| uIdtVector == X86_XCPT_PF))
4809	{
4810	enmReflect = VMXREFLECTXCPT_DF;
4811	}
4812	else if (uIdtVector == X86_XCPT_DF)
4813	enmReflect = VMXREFLECTXCPT_TF;
4814	}
4815	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4816	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4817	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4818	{
4819	/*
4820	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4821	* (whatever they are) as they reoccur when restarting the instruction.
4822	*/
4823	enmReflect = VMXREFLECTXCPT_XCPT;
4824	}
4825	}
4826	else
4827	{
4828	/*
4829	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4830	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4831	* original exception to the guest after handling the VM-exit.
4832	*/
4833	enmReflect = VMXREFLECTXCPT_XCPT;
4834	}
4835
4836	switch (enmReflect)
4837	{
4838	case VMXREFLECTXCPT_XCPT:
4839	{
4840	uint32_t u32ErrCode = 0;
4841	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4842	{
4843	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4844	AssertRCReturn(rc, rc);
4845	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4846	}
4847
4848	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4849	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4850	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4851	rc = VINF_SUCCESS;
4852	Log4(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4853	pVCpu->hm.s.Event.u32ErrCode));
4854	break;
4855	}
4856
4857	case VMXREFLECTXCPT_DF:
4858	{
4859	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4860	rc = VINF_HM_DOUBLE_FAULT;
4861	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4862	uIdtVector, uExitVector));
4863	break;
4864	}
4865
4866	case VMXREFLECTXCPT_TF:
4867	{
4868	rc = VINF_EM_RESET;
4869	Log4(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4870	break;
4871	}
4872
4873	default:
4874	Assert(rc == VINF_SUCCESS);
4875	break;
4876	}
4877	}
4878	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4879	return rc;
4880	}
4881
4882
4883	/**
4884	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4885	*
4886	* @returns VBox status code.
4887	* @param pVCpu Pointer to the VMCPU.
4888	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4889	* out-of-sync. Make sure to update the required fields
4890	* before using them.
4891	*
4892	* @remarks No-long-jump zone!!!
4893	*/
4894	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4895	{
4896	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4897	{
4898	uint32_t uVal = 0;
4899	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4900	AssertRCReturn(rc, rc);
4901	uint32_t uShadow = 0;
4902	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4903	AssertRCReturn(rc, rc);
4904
4905	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4906	CPUMSetGuestCR0(pVCpu, uVal);
4907	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4908	}
4909	return VINF_SUCCESS;
4910	}
4911
4912
4913	/**
4914	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4915	*
4916	* @returns VBox status code.
4917	* @param pVCpu Pointer to the VMCPU.
4918	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4919	* out-of-sync. Make sure to update the required fields
4920	* before using them.
4921	*
4922	* @remarks No-long-jump zone!!!
4923	*/
4924	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4925	{
4926	int rc = VINF_SUCCESS;
4927	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4928	{
4929	uint32_t uVal = 0;
4930	uint32_t uShadow = 0;
4931	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4932	AssertRCReturn(rc, rc);
4933	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4934	AssertRCReturn(rc, rc);
4935
4936	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4937	CPUMSetGuestCR4(pVCpu, uVal);
4938	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4939	}
4940	return rc;
4941	}
4942
4943
4944	/**
4945	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4946	*
4947	* @returns VBox status code.
4948	* @param pVCpu Pointer to the VMCPU.
4949	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4950	* out-of-sync. Make sure to update the required fields
4951	* before using them.
4952	*
4953	* @remarks No-long-jump zone!!!
4954	*/
4955	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4956	{
4957	int rc = VINF_SUCCESS;
4958	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4959	{
4960	uint64_t u64Val = 0;
4961	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4962	AssertRCReturn(rc, rc);
4963
4964	pMixedCtx->rip = u64Val;
4965	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4966	}
4967	return rc;
4968	}
4969
4970
4971	/**
4972	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4973	*
4974	* @returns VBox status code.
4975	* @param pVCpu Pointer to the VMCPU.
4976	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4977	* out-of-sync. Make sure to update the required fields
4978	* before using them.
4979	*
4980	* @remarks No-long-jump zone!!!
4981	*/
4982	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4983	{
4984	int rc = VINF_SUCCESS;
4985	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4986	{
4987	uint64_t u64Val = 0;
4988	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4989	AssertRCReturn(rc, rc);
4990
4991	pMixedCtx->rsp = u64Val;
4992	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4993	}
4994	return rc;
4995	}
4996
4997
4998	/**
4999	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5000	*
5001	* @returns VBox status code.
5002	* @param pVCpu Pointer to the VMCPU.
5003	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5004	* out-of-sync. Make sure to update the required fields
5005	* before using them.
5006	*
5007	* @remarks No-long-jump zone!!!
5008	*/
5009	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5010	{
5011	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5012	{
5013	uint32_t uVal = 0;
5014	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5015	AssertRCReturn(rc, rc);
5016
5017	pMixedCtx->eflags.u32 = uVal;
5018	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5019	{
5020	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5021	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5022
5023	pMixedCtx->eflags.Bits.u1VM = 0;
5024	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5025	}
5026
5027	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5028	}
5029	return VINF_SUCCESS;
5030	}
5031
5032
5033	/**
5034	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5035	* guest-CPU context.
5036	*/
5037	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5038	{
5039	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5040	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5041	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5042	return rc;
5043	}
5044
5045
5046	/**
5047	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5048	* from the guest-state area in the VMCS.
5049	*
5050	* @param pVCpu Pointer to the VMCPU.
5051	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5052	* out-of-sync. Make sure to update the required fields
5053	* before using them.
5054	*
5055	* @remarks No-long-jump zone!!!
5056	*/
5057	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5058	{
5059	uint32_t uIntrState = 0;
5060	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5061	AssertRC(rc);
5062
5063	if (!uIntrState)
5064	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5065	else
5066	{
5067	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5068	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5069	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5070	AssertRC(rc);
5071	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5072	AssertRC(rc);
5073
5074	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5075	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5076	}
5077	}
5078
5079
5080	/**
5081	* Saves the guest's activity state.
5082	*
5083	* @returns VBox status code.
5084	* @param pVCpu Pointer to the VMCPU.
5085	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5086	* out-of-sync. Make sure to update the required fields
5087	* before using them.
5088	*
5089	* @remarks No-long-jump zone!!!
5090	*/
5091	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5092	{
5093	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5094	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5095	return VINF_SUCCESS;
5096	}
5097
5098
5099	/**
5100	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5101	* the current VMCS into the guest-CPU context.
5102	*
5103	* @returns VBox status code.
5104	* @param pVCpu Pointer to the VMCPU.
5105	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5106	* out-of-sync. Make sure to update the required fields
5107	* before using them.
5108	*
5109	* @remarks No-long-jump zone!!!
5110	*/
5111	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5112	{
5113	int rc = VINF_SUCCESS;
5114	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5115	{
5116	uint32_t u32Val = 0;
5117	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5118	pMixedCtx->SysEnter.cs = u32Val;
5119	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5120	}
5121
5122	uint64_t u64Val = 0;
5123	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5124	{
5125	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5126	pMixedCtx->SysEnter.eip = u64Val;
5127	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5128	}
5129	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5130	{
5131	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5132	pMixedCtx->SysEnter.esp = u64Val;
5133	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5134	}
5135	return rc;
5136	}
5137
5138
5139	/**
5140	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5141	* context.
5142	*
5143	* @returns VBox status code.
5144	* @param pVCpu Pointer to the VMCPU.
5145	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5146	* out-of-sync. Make sure to update the required fields
5147	* before using them.
5148	*
5149	* @remarks No-long-jump zone!!!
5150	*/
5151	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5152	{
5153	int rc = VINF_SUCCESS;
5154	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5155	{
5156	uint64_t u64Val = 0;
5157	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5158	pMixedCtx->fs.u64Base = u64Val;
5159	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5160	}
5161	return rc;
5162	}
5163
5164
5165	/**
5166	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5167	* context.
5168	*
5169	* @returns VBox status code.
5170	* @param pVCpu Pointer to the VMCPU.
5171	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5172	* out-of-sync. Make sure to update the required fields
5173	* before using them.
5174	*
5175	* @remarks No-long-jump zone!!!
5176	*/
5177	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5178	{
5179	int rc = VINF_SUCCESS;
5180	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5181	{
5182	uint64_t u64Val = 0;
5183	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5184	pMixedCtx->gs.u64Base = u64Val;
5185	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5186	}
5187	return rc;
5188	}
5189
5190
5191	/**
5192	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5193	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5194	* and TSC_AUX.
5195	*
5196	* @returns VBox status code.
5197	* @param pVCpu Pointer to the VMCPU.
5198	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5199	* out-of-sync. Make sure to update the required fields
5200	* before using them.
5201	*
5202	* @remarks No-long-jump zone!!!
5203	*/
5204	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5205	{
5206	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5207	return VINF_SUCCESS;
5208
5209	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5210	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5211	{
5212	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5213	pMsr += i;
5214	switch (pMsr->u32IndexMSR)
5215	{
5216	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5217	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5218	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5219	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5220	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5221	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5222	default:
5223	{
5224	AssertFailed();
5225	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5226	}
5227	}
5228	}
5229	#endif
5230
5231	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5232	return VINF_SUCCESS;
5233	}
5234
5235
5236	/**
5237	* Saves the guest control registers from the current VMCS into the guest-CPU
5238	* context.
5239	*
5240	* @returns VBox status code.
5241	* @param pVCpu Pointer to the VMCPU.
5242	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5243	* out-of-sync. Make sure to update the required fields
5244	* before using them.
5245	*
5246	* @remarks No-long-jump zone!!!
5247	*/
5248	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5249	{
5250	/* Guest CR0. Guest FPU. */
5251	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5252	AssertRCReturn(rc, rc);
5253
5254	/* Guest CR4. */
5255	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5256	AssertRCReturn(rc, rc);
5257
5258	/* Guest CR2 - updated always during the world-switch or in #PF. */
5259	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5260	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5261	{
5262	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5263	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5264
5265	PVM pVM = pVCpu->CTX_SUFF(pVM);
5266	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5267	\|\| ( pVM->hm.s.fNestedPaging
5268	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5269	{
5270	uint64_t u64Val = 0;
5271	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5272	if (pMixedCtx->cr3 != u64Val)
5273	{
5274	CPUMSetGuestCR3(pVCpu, u64Val);
5275	if (VMMRZCallRing3IsEnabled(pVCpu))
5276	{
5277	PGMUpdateCR3(pVCpu, u64Val);
5278	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5279	}
5280	else
5281	{
5282	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5283	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5284	}
5285	}
5286
5287	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5288	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5289	{
5290	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5291	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5292	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5293	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5294
5295	if (VMMRZCallRing3IsEnabled(pVCpu))
5296	{
5297	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5298	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5299	}
5300	else
5301	{
5302	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5303	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5304	}
5305	}
5306	}
5307
5308	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5309	}
5310
5311	/*
5312	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5313	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5314	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5315	*
5316	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5317	*/
5318	if (VMMRZCallRing3IsEnabled(pVCpu))
5319	{
5320	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5321	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5322
5323	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5324	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5325
5326	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5327	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5328	}
5329
5330	return rc;
5331	}
5332
5333
5334	/**
5335	* Reads a guest segment register from the current VMCS into the guest-CPU
5336	* context.
5337	*
5338	* @returns VBox status code.
5339	* @param pVCpu Pointer to the VMCPU.
5340	* @param idxSel Index of the selector in the VMCS.
5341	* @param idxLimit Index of the segment limit in the VMCS.
5342	* @param idxBase Index of the segment base in the VMCS.
5343	* @param idxAccess Index of the access rights of the segment in the VMCS.
5344	* @param pSelReg Pointer to the segment selector.
5345	*
5346	* @remarks No-long-jump zone!!!
5347	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5348	* macro as that takes care of whether to read from the VMCS cache or
5349	* not.
5350	*/
5351	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5352	PCPUMSELREG pSelReg)
5353	{
5354	uint32_t u32Val = 0;
5355	int rc = VMXReadVmcs32(idxSel, &u32Val);
5356	AssertRCReturn(rc, rc);
5357	pSelReg->Sel = (uint16_t)u32Val;
5358	pSelReg->ValidSel = (uint16_t)u32Val;
5359	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5360
5361	rc = VMXReadVmcs32(idxLimit, &u32Val);
5362	AssertRCReturn(rc, rc);
5363	pSelReg->u32Limit = u32Val;
5364
5365	uint64_t u64Val = 0;
5366	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5367	AssertRCReturn(rc, rc);
5368	pSelReg->u64Base = u64Val;
5369
5370	rc = VMXReadVmcs32(idxAccess, &u32Val);
5371	AssertRCReturn(rc, rc);
5372	pSelReg->Attr.u = u32Val;
5373
5374	/*
5375	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5376	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5377	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5378	*
5379	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5380	*/
5381	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5382	{
5383	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5384	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5385	}
5386	return VINF_SUCCESS;
5387	}
5388
5389
5390	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5391	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5392	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5393	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5394	#else
5395	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5396	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5397	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5398	#endif
5399
5400
5401	/**
5402	* Saves the guest segment registers from the current VMCS into the guest-CPU
5403	* context.
5404	*
5405	* @returns VBox status code.
5406	* @param pVCpu Pointer to the VMCPU.
5407	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5408	* out-of-sync. Make sure to update the required fields
5409	* before using them.
5410	*
5411	* @remarks No-long-jump zone!!!
5412	*/
5413	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5414	{
5415	/* Guest segment registers. */
5416	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5417	{
5418	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5419	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5420	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5421	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5422	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5423	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5424	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5425
5426	/* Restore segment attributes for real-on-v86 mode hack. */
5427	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5428	{
5429	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5430	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5431	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5432	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5433	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5434	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5435	}
5436	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5437	}
5438
5439	return VINF_SUCCESS;
5440	}
5441
5442
5443	/**
5444	* Saves the guest descriptor table registers and task register from the current
5445	* VMCS into the guest-CPU context.
5446	*
5447	* @returns VBox status code.
5448	* @param pVCpu Pointer to the VMCPU.
5449	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5450	* out-of-sync. Make sure to update the required fields
5451	* before using them.
5452	*
5453	* @remarks No-long-jump zone!!!
5454	*/
5455	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5456	{
5457	int rc = VINF_SUCCESS;
5458
5459	/* Guest LDTR. */
5460	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5461	{
5462	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5463	AssertRCReturn(rc, rc);
5464	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5465	}
5466
5467	/* Guest GDTR. */
5468	uint64_t u64Val = 0;
5469	uint32_t u32Val = 0;
5470	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5471	{
5472	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5473	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5474	pMixedCtx->gdtr.pGdt = u64Val;
5475	pMixedCtx->gdtr.cbGdt = u32Val;
5476	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5477	}
5478
5479	/* Guest IDTR. */
5480	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5481	{
5482	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5483	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5484	pMixedCtx->idtr.pIdt = u64Val;
5485	pMixedCtx->idtr.cbIdt = u32Val;
5486	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5487	}
5488
5489	/* Guest TR. */
5490	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5491	{
5492	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5493	AssertRCReturn(rc, rc);
5494
5495	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5496	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5497	{
5498	rc = VMXLOCAL_READ_SEG(TR, tr);
5499	AssertRCReturn(rc, rc);
5500	}
5501	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5502	}
5503	return rc;
5504	}
5505
5506	#undef VMXLOCAL_READ_SEG
5507
5508
5509	/**
5510	* Saves the guest debug registers from the current VMCS into the guest-CPU
5511	* context.
5512	*
5513	* @returns VBox status code.
5514	* @param pVCpu Pointer to the VMCPU.
5515	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5516	* out-of-sync. Make sure to update the required fields
5517	* before using them.
5518	*
5519	* @remarks No-long-jump zone!!!
5520	*/
5521	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5522	{
5523	int rc = VINF_SUCCESS;
5524	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5525	{
5526	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5527	uint32_t u32Val;
5528	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5529	pMixedCtx->dr[7] = u32Val;
5530
5531	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5532	}
5533	return rc;
5534	}
5535
5536
5537	/**
5538	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5539	*
5540	* @returns VBox status code.
5541	* @param pVCpu Pointer to the VMCPU.
5542	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5543	* out-of-sync. Make sure to update the required fields
5544	* before using them.
5545	*
5546	* @remarks No-long-jump zone!!!
5547	*/
5548	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5549	{
5550	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5551	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5552	return VINF_SUCCESS;
5553	}
5554
5555
5556	/**
5557	* Saves the entire guest state from the currently active VMCS into the
5558	* guest-CPU context. This essentially VMREADs all guest-data.
5559	*
5560	* @returns VBox status code.
5561	* @param pVCpu Pointer to the VMCPU.
5562	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5563	* out-of-sync. Make sure to update the required fields
5564	* before using them.
5565	*/
5566	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5567	{
5568	Assert(pVCpu);
5569	Assert(pMixedCtx);
5570
5571	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5572	return VINF_SUCCESS;
5573
5574	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5575	there is no real need to. */
5576	if (VMMRZCallRing3IsEnabled(pVCpu))
5577	VMMR0LogFlushDisable(pVCpu);
5578	else
5579	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5580	Log4Func(("\n"));
5581
5582	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5583	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5584
5585	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5586	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5587
5588	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5589	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5590
5591	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5592	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5593
5594	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5595	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5596
5597	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5598	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5599
5600	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5601	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5602
5603	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5604	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5605
5606	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5607	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5608
5609	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5610	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5611
5612	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5613	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5614
5615	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5616	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5617
5618	if (VMMRZCallRing3IsEnabled(pVCpu))
5619	VMMR0LogFlushEnable(pVCpu);
5620
5621	return rc;
5622	}
5623
5624
5625	/**
5626	* Check per-VM and per-VCPU force flag actions that require us to go back to
5627	* ring-3 for one reason or another.
5628	*
5629	* @returns VBox status code (information status code included).
5630	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5631	* ring-3.
5632	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5633	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5634	* interrupts)
5635	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5636	* all EMTs to be in ring-3.
5637	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5638	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5639	* to the EM loop.
5640	*
5641	* @param pVM Pointer to the VM.
5642	* @param pVCpu Pointer to the VMCPU.
5643	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5644	* out-of-sync. Make sure to update the required fields
5645	* before using them.
5646	*/
5647	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5648	{
5649	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5650
5651	int rc = VERR_INTERNAL_ERROR_5;
5652	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5653	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5654	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5655	{
5656	/* We need the control registers now, make sure the guest-CPU context is updated. */
5657	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5658	AssertRCReturn(rc, rc);
5659
5660	/* Pending HM CR3 sync. */
5661	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5662	{
5663	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5664	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5665	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5666	}
5667
5668	/* Pending HM PAE PDPEs. */
5669	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5670	{
5671	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5672	AssertRC(rc);
5673	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5674	}
5675
5676	/* Pending PGM C3 sync. */
5677	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5678	{
5679	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5680	if (rc != VINF_SUCCESS)
5681	{
5682	AssertRC(rc);
5683	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5684	return rc;
5685	}
5686	}
5687
5688	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5689	/* -XXX- what was that about single stepping? */
5690	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5691	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5692	{
5693	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5694	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5695	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5696	return rc;
5697	}
5698
5699	/* Pending VM request packets, such as hardware interrupts. */
5700	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5701	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5702	{
5703	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5704	return VINF_EM_PENDING_REQUEST;
5705	}
5706
5707	/* Pending PGM pool flushes. */
5708	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5709	{
5710	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5711	return VINF_PGM_POOL_FLUSH_PENDING;
5712	}
5713
5714	/* Pending DMA requests. */
5715	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5716	{
5717	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5718	return VINF_EM_RAW_TO_R3;
5719	}
5720	}
5721
5722	/* Paranoia. */
5723	Assert(rc != VERR_EM_INTERPRETER);
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	/**
5729	* Converts any TRPM trap into a pending VMX event. This is typically used when
5730	* entering from ring-3 (not longjmp returns).
5731	*
5732	* @param pVCpu Pointer to the VMCPU.
5733	*/
5734	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5735	{
5736	Assert(TRPMHasTrap(pVCpu));
5737	Assert(!pVCpu->hm.s.Event.fPending);
5738
5739	uint8_t uVector;
5740	TRPMEVENT enmTrpmEvent;
5741	RTGCUINT uErrCode;
5742	RTGCUINTPTR GCPtrFaultAddress;
5743	uint8_t cbInstr;
5744
5745	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5746	AssertRC(rc);
5747
5748	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5749	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5750	if (enmTrpmEvent == TRPM_TRAP)
5751	{
5752	switch (uVector)
5753	{
5754	case X86_XCPT_BP:
5755	case X86_XCPT_OF:
5756	{
5757	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5758	break;
5759	}
5760
5761	case X86_XCPT_PF:
5762	case X86_XCPT_DF:
5763	case X86_XCPT_TS:
5764	case X86_XCPT_NP:
5765	case X86_XCPT_SS:
5766	case X86_XCPT_GP:
5767	case X86_XCPT_AC:
5768	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5769	/* no break! */
5770	default:
5771	{
5772	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5773	break;
5774	}
5775	}
5776	}
5777	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5778	{
5779	if (uVector == X86_XCPT_NMI)
5780	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5781	else
5782	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5783	}
5784	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5785	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5786	else
5787	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5788
5789	rc = TRPMResetTrap(pVCpu);
5790	AssertRC(rc);
5791	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5792	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5793	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5794	}
5795
5796
5797	/**
5798	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5799	* VT-x to execute any instruction.
5800	*
5801	* @param pvCpu Pointer to the VMCPU.
5802	*/
5803	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5804	{
5805	Assert(pVCpu->hm.s.Event.fPending);
5806
5807	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5808	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5809	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5810	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5811
5812	/* If a trap was already pending, we did something wrong! */
5813	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5814
5815	TRPMEVENT enmTrapType;
5816	switch (uVectorType)
5817	{
5818	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5819	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5820	enmTrapType = TRPM_HARDWARE_INT;
5821	break;
5822	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5823	enmTrapType = TRPM_SOFTWARE_INT;
5824	break;
5825	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5826	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5827	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5828	enmTrapType = TRPM_TRAP;
5829	break;
5830	default:
5831	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5832	enmTrapType = TRPM_32BIT_HACK;
5833	break;
5834	}
5835
5836	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5837
5838	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5839	AssertRC(rc);
5840
5841	if (fErrorCodeValid)
5842	TRPMSetErrorCode(pVCpu, uErrorCode);
5843
5844	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5845	&& uVector == X86_XCPT_PF)
5846	{
5847	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5848	}
5849	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5850	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5851	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5852	{
5853	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5854	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5855	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5856	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5857	}
5858	pVCpu->hm.s.Event.fPending = false;
5859	}
5860
5861
5862	/**
5863	* Does the necessary state syncing before doing a longjmp to ring-3.
5864	*
5865	* @param pVM Pointer to the VM.
5866	* @param pVCpu Pointer to the VMCPU.
5867	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5868	* out-of-sync. Make sure to update the required fields
5869	* before using them.
5870	* @param rcExit The reason for exiting to ring-3. Can be
5871	* VINF_VMM_UNKNOWN_RING3_CALL.
5872	*
5873	* @remarks No-long-jmp zone!!!
5874	*/
5875	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5876	{
5877	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5878	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5879
5880	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5881	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5882	AssertRC(rc);
5883
5884	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
5885	if (CPUMIsGuestFPUStateActive(pVCpu))
5886	{
5887	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5888	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5889	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5890	}
5891
5892	/* Restore host debug registers if necessary and resync on next R0 reentry. */
5893	if (CPUMIsGuestDebugStateActive(pVCpu))
5894	{
5895	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5896	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5897	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5898	}
5899	else if (CPUMIsHyperDebugStateActive(pVCpu))
5900	{
5901	CPUMR0LoadHostDebugState(pVM, pVCpu);
5902	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5903	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5904	}
5905
5906	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5907	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
5908	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5909	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5910	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5911	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5912	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5913	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5914	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5915	}
5916
5917
5918	/**
5919	* An action requires us to go back to ring-3. This function does the necessary
5920	* steps before we can safely return to ring-3. This is not the same as longjmps
5921	* to ring-3, this is voluntary.
5922	*
5923	* @param pVM Pointer to the VM.
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	* @param rcExit The reason for exiting to ring-3. Can be
5929	* VINF_VMM_UNKNOWN_RING3_CALL.
5930	*/
5931	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5932	{
5933	Assert(pVM);
5934	Assert(pVCpu);
5935	Assert(pMixedCtx);
5936	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5937
5938	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5939	{
5940	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5941	return;
5942	}
5943	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5944	{
5945	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5946	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5947	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5948	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5949	return;
5950	}
5951
5952	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5953	VMMRZCallRing3Disable(pVCpu);
5954	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
5955
5956	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5957	if (pVCpu->hm.s.Event.fPending)
5958	{
5959	hmR0VmxPendingEventToTrpmTrap(pVCpu);
5960	Assert(!pVCpu->hm.s.Event.fPending);
5961	}
5962
5963	/* Sync. the guest state. */
5964	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5965	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5966
5967	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5968	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5969	\| CPUM_CHANGED_LDTR
5970	\| CPUM_CHANGED_GDTR
5971	\| CPUM_CHANGED_IDTR
5972	\| CPUM_CHANGED_TR
5973	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5974
5975	/* On our way back from ring-3 the following needs to be done. */
5976	/** @todo This can change with preemption hooks. */
5977	if (rcExit == VINF_EM_RAW_INTERRUPT)
5978	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5979	else
5980	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5981
5982	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5983	VMMRZCallRing3Enable(pVCpu);
5984	}
5985
5986
5987	/**
5988	* VMMRZCallRing3() callback wrapper which saves the guest state before we
5989	* longjump to ring-3 and possibly get preempted.
5990	*
5991	* @param pVCpu Pointer to the VMCPU.
5992	* @param enmOperation The operation causing the ring-3 longjump.
5993	* @param pvUser The user argument (pointer to the possibly
5994	* out-of-date guest-CPU context).
5995	*
5996	* @remarks Must never be called with @a enmOperation ==
5997	* VMMCALLRING3_VM_R0_ASSERTION.
5998	*/
5999	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6000	{
6001	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6002	Assert(pVCpu);
6003	Assert(pvUser);
6004	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6005	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6006
6007	VMMRZCallRing3Disable(pVCpu);
6008	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6009	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6010	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6011	VMMRZCallRing3Enable(pVCpu);
6012	}
6013
6014
6015	/**
6016	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6017	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6018	*
6019	* @param pVCpu Pointer to the VMCPU.
6020	*/
6021	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6022	{
6023	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6024	{
6025	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6026	{
6027	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6028	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6029	AssertRC(rc);
6030	}
6031	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6032	}
6033
6034
6035	/**
6036	* Injects any pending events into the guest if the guest is in a state to
6037	* receive them.
6038	*
6039	* @returns VBox status code (informational status codes included).
6040	* @param pVCpu Pointer to the VMCPU.
6041	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6042	* out-of-sync. Make sure to update the required fields
6043	* before using them.
6044	*/
6045	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6046	{
6047	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6048	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6049	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6050	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6051
6052	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6053	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6054	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6055	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6056	Assert(!TRPMHasTrap(pVCpu));
6057
6058	int rc = VINF_SUCCESS;
6059	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6060	{
6061	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6062	bool fInject = true;
6063	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6064	{
6065	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6066	AssertRCReturn(rc, rc);
6067	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6068	if ( fBlockInt
6069	\|\| fBlockSti
6070	\|\| fBlockMovSS)
6071	{
6072	fInject = false;
6073	}
6074	}
6075	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6076	&& ( fBlockMovSS
6077	\|\| fBlockSti))
6078	{
6079	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6080	fInject = false;
6081	}
6082
6083	if (fInject)
6084	{
6085	Log4(("Injecting pending event\n"));
6086	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6087	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6088	AssertRCReturn(rc, rc);
6089	pVCpu->hm.s.Event.fPending = false;
6090	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
6091	}
6092	else
6093	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6094	} /** @todo SMI. SMIs take priority over NMIs. */
6095	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6096	{
6097	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6098	if ( !fBlockMovSS
6099	&& !fBlockSti)
6100	{
6101	Log4(("Injecting NMI\n"));
6102	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6103	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6104	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6105	0 /* GCPtrFaultAddress */, &uIntrState);
6106	AssertRCReturn(rc, rc);
6107	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6108	}
6109	else
6110	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6111	}
6112	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6113	{
6114	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6115	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6116	AssertRCReturn(rc, rc);
6117	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6118	if ( !fBlockInt
6119	&& !fBlockSti
6120	&& !fBlockMovSS)
6121	{
6122	uint8_t u8Interrupt;
6123	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6124	if (RT_SUCCESS(rc))
6125	{
6126	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6127	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6128	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6129	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6130	0 /* GCPtrFaultAddress */, &uIntrState);
6131	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6132	}
6133	else
6134	{
6135	/** @todo Does this actually happen? If not turn it into an assertion. */
6136	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6137	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6138	rc = VINF_SUCCESS;
6139	}
6140	}
6141	else
6142	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6143	}
6144
6145	/*
6146	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6147	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6148	*/
6149	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6150	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6151	int rc2 = VINF_SUCCESS;
6152	if ( fBlockSti
6153	\|\| fBlockMovSS)
6154	{
6155	if (!DBGFIsStepping(pVCpu))
6156	{
6157	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6158	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6159	{
6160	/*
6161	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6162	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6163	*/
6164	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6165	AssertRCReturn(rc, rc);
6166	}
6167	}
6168	else
6169	{
6170	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6171	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6172	uIntrState = 0;
6173	}
6174	}
6175
6176	/*
6177	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6178	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6179	*/
6180	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6181	AssertRC(rc2);
6182
6183	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6184	return rc;
6185	}
6186
6187
6188	/**
6189	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6190	*
6191	* @param pVCpu Pointer to the VMCPU.
6192	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6193	* out-of-sync. Make sure to update the required fields
6194	* before using them.
6195	*/
6196	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6197	{
6198	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6199	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6200	}
6201
6202
6203	/**
6204	* Injects a double-fault (#DF) exception into the VM.
6205	*
6206	* @returns VBox status code (informational status code included).
6207	* @param pVCpu Pointer to the VMCPU.
6208	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6209	* out-of-sync. Make sure to update the required fields
6210	* before using them.
6211	*/
6212	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6213	{
6214	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6215	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6216	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6217	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6218	puIntrState);
6219	}
6220
6221
6222	/**
6223	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6224	*
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	*/
6230	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6231	{
6232	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6233	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6234	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6235	}
6236
6237
6238	/**
6239	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6240	*
6241	* @param pVCpu Pointer to the VMCPU.
6242	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6243	* out-of-sync. Make sure to update the required fields
6244	* before using them.
6245	* @param cbInstr The value of RIP that is to be pushed on the guest
6246	* stack.
6247	*/
6248	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6249	{
6250	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6251	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6252	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6253	}
6254
6255
6256	/**
6257	* Injects a general-protection (#GP) fault into the VM.
6258	*
6259	* @returns VBox status code (informational status code included).
6260	* @param pVCpu Pointer to the VMCPU.
6261	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6262	* out-of-sync. Make sure to update the required fields
6263	* before using them.
6264	* @param u32ErrorCode The error code associated with the #GP.
6265	*/
6266	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6267	uint32_t *puIntrState)
6268	{
6269	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6270	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6271	if (fErrorCodeValid)
6272	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6273	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6274	puIntrState);
6275	}
6276
6277
6278	/**
6279	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6280	*
6281	* @param pVCpu Pointer to the VMCPU.
6282	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6283	* out-of-sync. Make sure to update the required fields
6284	* before using them.
6285	* @param uVector The software interrupt vector number.
6286	* @param cbInstr The value of RIP that is to be pushed on the guest
6287	* stack.
6288	*/
6289	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6290	{
6291	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6292	if ( uVector == X86_XCPT_BP
6293	\|\| uVector == X86_XCPT_OF)
6294	{
6295	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6296	}
6297	else
6298	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6299	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6300	}
6301
6302
6303	/**
6304	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6305	* stack.
6306	*
6307	* @returns VBox status code (information status code included).
6308	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6309	* @param pVM Pointer to the VM.
6310	* @param pMixedCtx Pointer to the guest-CPU context.
6311	* @param uValue The value to push to the guest stack.
6312	*/
6313	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6314	{
6315	/*
6316	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6317	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6318	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6319	*/
6320	if (pMixedCtx->sp == 1)
6321	return VINF_EM_RESET;
6322	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6323	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6324	AssertRCReturn(rc, rc);
6325	return rc;
6326	}
6327
6328
6329	/**
6330	* Injects an event into the guest upon VM-entry by updating the relevant fields
6331	* in the VM-entry area in the VMCS.
6332	*
6333	* @returns VBox status code (informational error codes included).
6334	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6335	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6336	*
6337	* @param pVCpu Pointer to the VMCPU.
6338	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6339	* be out-of-sync. Make sure to update the required
6340	* fields before using them.
6341	* @param u64IntrInfo The VM-entry interruption-information field.
6342	* @param cbInstr The VM-entry instruction length in bytes (for
6343	* software interrupts, exceptions and privileged
6344	* software exceptions).
6345	* @param u32ErrCode The VM-entry exception error code.
6346	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6347	* @param puIntrState Pointer to the current guest interruptibility-state.
6348	* This interruptibility-state will be updated if
6349	* necessary. This cannot not be NULL.
6350	*
6351	* @remarks No-long-jump zone!!!
6352	* @remarks Requires CR0!
6353	*/
6354	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6355	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6356	{
6357	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6358	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6359	Assert(puIntrState);
6360	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6361
6362	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6363	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6364
6365	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6366	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6367	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6368
6369	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6370
6371	/* We require CR0 to check if the guest is in real-mode. */
6372	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6373	AssertRCReturn(rc, rc);
6374
6375	/*
6376	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6377	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6378	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6379	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6380	*/
6381	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6382	{
6383	PVM pVM = pVCpu->CTX_SUFF(pVM);
6384	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6385	{
6386	Assert(PDMVmmDevHeapIsEnabled(pVM));
6387	Assert(pVM->hm.s.vmx.pRealModeTSS);
6388
6389	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6390	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6391	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6392	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6393	AssertRCReturn(rc, rc);
6394	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6395
6396	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6397	const size_t cbIdtEntry = 4;
6398	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6399	{
6400	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6401	if (uVector == X86_XCPT_DF)
6402	return VINF_EM_RESET;
6403	else if (uVector == X86_XCPT_GP)
6404	{
6405	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6406	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6407	}
6408
6409	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6410	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6411	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6412	}
6413
6414	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6415	uint16_t uGuestIp = pMixedCtx->ip;
6416	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6417	{
6418	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6419	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6420	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6421	}
6422	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6423	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6424
6425	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6426	uint16_t offIdtEntry = 0;
6427	RTSEL selIdtEntry = 0;
6428	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6429	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6430	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6431	AssertRCReturn(rc, rc);
6432
6433	/* Construct the stack frame for the interrupt/exception handler. */
6434	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6435	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6436	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6437	AssertRCReturn(rc, rc);
6438
6439	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6440	if (rc == VINF_SUCCESS)
6441	{
6442	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6443	pMixedCtx->rip = offIdtEntry;
6444	pMixedCtx->cs.Sel = selIdtEntry;
6445	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6446	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6447	&& uVector == X86_XCPT_PF)
6448	{
6449	pMixedCtx->cr2 = GCPtrFaultAddress;
6450	}
6451	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6452	\| HM_CHANGED_GUEST_RIP
6453	\| HM_CHANGED_GUEST_RFLAGS
6454	\| HM_CHANGED_GUEST_RSP;
6455
6456	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6457	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6458	{
6459	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6460	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6461	Log4(("Clearing inhibition due to STI.\n"));
6462	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6463	}
6464	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6465	}
6466	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6467	return rc;
6468	}
6469	else
6470	{
6471	/*
6472	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6473	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6474	*/
6475	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6476	}
6477	}
6478
6479	/* Validate. */
6480	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6481	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6482	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6483
6484	/* Inject. */
6485	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6486	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6487	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6488	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6489
6490	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6491	&& uVector == X86_XCPT_PF)
6492	{
6493	pMixedCtx->cr2 = GCPtrFaultAddress;
6494	}
6495
6496	Log4(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr,
6497	pMixedCtx->cr2));
6498
6499	AssertRCReturn(rc, rc);
6500	return rc;
6501	}
6502
6503
6504	/**
6505	* Enters the VT-x session.
6506	*
6507	* @returns VBox status code.
6508	* @param pVM Pointer to the VM.
6509	* @param pVCpu Pointer to the VMCPU.
6510	* @param pCpu Pointer to the CPU info struct.
6511	*/
6512	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6513	{
6514	AssertPtr(pVM);
6515	AssertPtr(pVCpu);
6516	Assert(pVM->hm.s.vmx.fSupported);
6517	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6518	NOREF(pCpu);
6519
6520	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6521
6522	/* Make sure we're in VMX root mode. */
6523	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6524	if (!(u32HostCR4 & X86_CR4_VMXE))
6525	{
6526	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6527	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6528	}
6529
6530	/* Load the active VMCS as the current one. */
6531	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6532	if (RT_FAILURE(rc))
6533	return rc;
6534
6535	/** @todo this will change with preemption hooks where can can VMRESUME as long
6536	* as we're no preempted. */
6537	pVCpu->hm.s.fResumeVM = false;
6538	return VINF_SUCCESS;
6539	}
6540
6541
6542	/**
6543	* Leaves the VT-x session.
6544	*
6545	* @returns VBox status code.
6546	* @param pVM Pointer to the VM.
6547	* @param pVCpu Pointer to the VMCPU.
6548	* @param pCtx Pointer to the guest-CPU context.
6549	*/
6550	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6551	{
6552	AssertPtr(pVCpu);
6553	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6554	NOREF(pVM);
6555	NOREF(pCtx);
6556
6557	/** @todo this will change with preemption hooks where we only VMCLEAR when
6558	* we are actually going to be preempted, not all the time like we
6559	* currently do. */
6560
6561	/* Restore host-state bits that VT-x only restores partially. */
6562	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6563	{
6564	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6565	/** @todo r=ramshankar: This is broken when
6566	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6567	* VMXRestoreHostState() may unconditionally enables interrupts. */
6568	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6569	#else
6570	Assert(ASMIntAreEnabled());
6571	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6572	#endif
6573	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6574	}
6575
6576	/*
6577	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6578	* and mark the VMCS launch-state as "clear".
6579	*/
6580	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6581	return rc;
6582	}
6583
6584
6585	/**
6586	* Saves the host state in the VMCS host-state.
6587	* Sets up the VM-exit MSR-load area.
6588	*
6589	* The CPU state will be loaded from these fields on every successful VM-exit.
6590	*
6591	* @returns VBox status code.
6592	* @param pVM Pointer to the VM.
6593	* @param pVCpu Pointer to the VMCPU.
6594	*
6595	* @remarks No-long-jump zone!!!
6596	*/
6597	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6598	{
6599	AssertPtr(pVM);
6600	AssertPtr(pVCpu);
6601	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6602
6603	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6604
6605	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6606	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6607	return VINF_SUCCESS;
6608
6609	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6610	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6611
6612	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6613	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6614
6615	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6616	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6617
6618	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6619	return rc;
6620	}
6621
6622
6623	/**
6624	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6625	* loaded from these fields on every successful VM-entry.
6626	*
6627	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6628	* Sets up the VM-entry controls.
6629	* Sets up the appropriate VMX non-root function to execute guest code based on
6630	* the guest CPU mode.
6631	*
6632	* @returns VBox status code.
6633	* @param pVM Pointer to the VM.
6634	* @param pVCpu Pointer to the VMCPU.
6635	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6636	* out-of-sync. Make sure to update the required fields
6637	* before using them.
6638	*
6639	* @remarks No-long-jump zone!!!
6640	*/
6641	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6642	{
6643	AssertPtr(pVM);
6644	AssertPtr(pVCpu);
6645	AssertPtr(pMixedCtx);
6646	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6647
6648	#ifdef LOG_ENABLED
6649	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6650	* probably not initialized yet? Anyway this will do for now. */
6651	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6652	VMMR0LogFlushDisable(pVCpu);
6653	#endif
6654
6655	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6656
6657	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6658
6659	/* Determine real-on-v86 mode. */
6660	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6661	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6662	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6663	{
6664	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6665	}
6666
6667	/*
6668	* Load the guest-state into the VMCS.
6669	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6670	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6671	*/
6672	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6673	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6674
6675	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6676	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6677
6678	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6679	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6680
6681	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6682	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6683
6684	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6685	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6686	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6687
6688	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6689	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6690
6691	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6692	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6693
6694	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6695	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6696
6697	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6698	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6699	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6700
6701	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6702	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6703
6704	/* Clear any unused and reserved bits. */
6705	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
6706
6707	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6708	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6709	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6710
6711	#ifdef LOG_ENABLED
6712	/* Only reenable log-flushing if the caller has it enabled. */
6713	if (!fCallerDisabledLogFlush)
6714	VMMR0LogFlushEnable(pVCpu);
6715	#endif
6716
6717	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6718	return rc;
6719	}
6720
6721
6722	/**
6723	* Does the preparations before executing guest code in VT-x.
6724	*
6725	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6726	* recompiler. We must be cautious what we do here regarding committing
6727	* guest-state information into the the VMCS assuming we assuredly execute the
6728	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6729	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6730	* that the recompiler can (and should) use them when it resumes guest
6731	* execution. Otherwise such operations must be done when we can no longer
6732	* exit to ring-3.
6733	*
6734	* @returns VBox status code (informational status codes included).
6735	* @retval VINF_SUCCESS if we can proceed with running the guest.
6736	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6737	* into the guest.
6738	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6739	*
6740	* @param pVM Pointer to the VM.
6741	* @param pVCpu Pointer to the VMCPU.
6742	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6743	* out-of-sync. Make sure to update the required fields
6744	* before using them.
6745	* @param pVmxTransient Pointer to the VMX transient structure.
6746	*
6747	* @remarks Called with preemption disabled.
6748	*/
6749	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6750	{
6751	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6752
6753	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6754	PGMRZDynMapFlushAutoSet(pVCpu);
6755	#endif
6756
6757	/* Check force flag actions that might require us to go back to ring-3. */
6758	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6759	if (rc != VINF_SUCCESS)
6760	return rc;
6761
6762	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6763	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6764	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6765	{
6766	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6767	RTGCPHYS GCPhysApicBase;
6768	GCPhysApicBase = pMixedCtx->msrApicBase;
6769	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6770
6771	/* Unalias any existing mapping. */
6772	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6773	AssertRCReturn(rc, rc);
6774
6775	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6776	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6777	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6778	AssertRCReturn(rc, rc);
6779
6780	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6781	}
6782
6783	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6784	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6785	pVmxTransient->uEFlags = ASMIntDisableFlags();
6786	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6787	{
6788	ASMSetFlags(pVmxTransient->uEFlags);
6789	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6790	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6791	return VINF_EM_RAW_INTERRUPT;
6792	}
6793	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6794	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6795	#endif
6796
6797	/*
6798	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6799	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6800	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6801	*/
6802	/** @todo Rework event evaluation and injection to be completely separate. */
6803	if (TRPMHasTrap(pVCpu))
6804	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6805
6806	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6807	AssertRCReturn(rc, rc);
6808	return rc;
6809	}
6810
6811
6812	/**
6813	* Prepares to run guest code in VT-x and we've committed to doing so. This
6814	* means there is no backing out to ring-3 or anywhere else at this
6815	* point.
6816	*
6817	* @param pVM Pointer to the VM.
6818	* @param pVCpu Pointer to the VMCPU.
6819	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6820	* out-of-sync. Make sure to update the required fields
6821	* before using them.
6822	* @param pVmxTransient Pointer to the VMX transient structure.
6823	*
6824	* @remarks Called with preemption disabled.
6825	* @remarks No-long-jump zone!!!
6826	*/
6827	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6828	{
6829	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6830	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6831
6832	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6833	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6834	pVmxTransient->uEFlags = ASMIntDisableFlags();
6835	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6836	#endif
6837
6838	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6839	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6840	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6841	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6842	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6843	#endif
6844	int rc = VINF_SUCCESS;
6845	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6846	{
6847	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6848	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6849	}
6850	else if (pVCpu->hm.s.fContextUseFlags)
6851	{
6852	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6853	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6854	}
6855	AssertRC(rc);
6856	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6857
6858	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6859	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6860	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6861
6862	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6863	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6864	{
6865	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6866	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6867	}
6868
6869	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6870	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6871	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6872
6873	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6874	/*
6875	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6876	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6877	*/
6878	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6879	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6880	{
6881	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6882	uint64_t u64HostTscAux = 0;
6883	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6884	AssertRC(rc2);
6885	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6886	}
6887	#endif
6888
6889	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6890	to start executing. */
6891	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6892	}
6893
6894
6895	/**
6896	* Performs some essential restoration of state after running guest code in
6897	* VT-x.
6898	*
6899	* @param pVM Pointer to the VM.
6900	* @param pVCpu Pointer to the VMCPU.
6901	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6902	* out-of-sync. Make sure to update the required fields
6903	* before using them.
6904	* @param pVmxTransient Pointer to the VMX transient structure.
6905	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6906	*
6907	* @remarks Called with interrupts disabled.
6908	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6909	* unconditionally when it is safe to do so.
6910	*/
6911	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6912	{
6913	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6914	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6915
6916	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6917	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6918	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6919	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6920	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6921
6922	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6923	{
6924	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6925	/* Restore host's TSC_AUX. */
6926	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6927	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6928	#endif
6929	/** @todo Find a way to fix hardcoding a guestimate. */
6930	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6931	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6932	}
6933
6934	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6935	Assert(!(ASMGetFlags() & X86_EFL_IF));
6936	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6937
6938	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6939	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6940
6941	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6942	uint32_t uExitReason;
6943	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6944	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6945	AssertRC(rc);
6946	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6947	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6948
6949	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6950	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6951
6952	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6953	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6954	{
6955	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
6956	pVmxTransient->fVMEntryFailed));
6957	return;
6958	}
6959
6960	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6961	{
6962	/* Update the guest interruptibility-state from the VMCS. */
6963	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6964	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6965	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6966	AssertRC(rc);
6967	#endif
6968	/*
6969	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6970	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6971	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6972	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6973	*/
6974	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6975	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6976	{
6977	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6978	AssertRC(rc);
6979	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6980	}
6981	}
6982	}
6983
6984
6985	/**
6986	* Runs the guest code using VT-x.
6987	*
6988	* @returns VBox status code.
6989	* @param pVM Pointer to the VM.
6990	* @param pVCpu Pointer to the VMCPU.
6991	* @param pCtx Pointer to the guest-CPU context.
6992	*
6993	* @remarks Called with preemption disabled.
6994	*/
6995	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6996	{
6997	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6998	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6999
7000	VMXTRANSIENT VmxTransient;
7001	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7002	int rc = VERR_INTERNAL_ERROR_5;
7003	uint32_t cLoops = 0;
7004
7005	for (;; cLoops++)
7006	{
7007	Assert(!HMR0SuspendPending());
7008	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7009	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7010	(unsigned)RTMpCpuId(), cLoops));
7011
7012	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7013	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7014	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7015	if (rc != VINF_SUCCESS)
7016	break;
7017
7018	/*
7019	* No longjmps to ring-3 from this point on!!!
7020	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7021	* This also disables flushing of the R0-logger instance (if any).
7022	*/
7023	VMMRZCallRing3Disable(pVCpu);
7024	VMMRZCallRing3RemoveNotification(pVCpu);
7025	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7026
7027	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7028	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7029
7030	/*
7031	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7032	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7033	*/
7034	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7035	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7036	{
7037	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7038	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7039	return rc;
7040	}
7041
7042	/* Handle the VM-exit. */
7043	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7044	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7045	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7046	HMVMX_START_EXIT_DISPATCH_PROF();
7047	#ifdef HMVMX_USE_FUNCTION_TABLE
7048	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7049	#else
7050	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7051	#endif
7052	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7053	if (rc != VINF_SUCCESS)
7054	break;
7055	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7056	{
7057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7058	rc = VINF_EM_RAW_INTERRUPT;
7059	break;
7060	}
7061	}
7062
7063	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7064	if (rc == VERR_EM_INTERPRETER)
7065	rc = VINF_EM_RAW_EMULATE_INSTR;
7066	else if (rc == VINF_EM_RESET)
7067	rc = VINF_EM_TRIPLE_FAULT;
7068	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7069	return rc;
7070	}
7071
7072
7073	#ifndef HMVMX_USE_FUNCTION_TABLE
7074	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7075	{
7076	int rc;
7077	switch (rcReason)
7078	{
7079	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7080	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7081	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7082	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7083	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7084	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7085	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7086	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7087	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7088	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7089	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7090	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7091	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7092	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7093	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7094	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7095	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7096	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7097	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7098	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7099	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7100	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7101	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7102	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7103	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7104	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7105	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7106	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7107	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7108	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7109	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7110	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7111	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7112
7113	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7114	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7115	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7116	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7117	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7118	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7119	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7120	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7121	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7122
7123	case VMX_EXIT_VMCALL:
7124	case VMX_EXIT_VMCLEAR:
7125	case VMX_EXIT_VMLAUNCH:
7126	case VMX_EXIT_VMPTRLD:
7127	case VMX_EXIT_VMPTRST:
7128	case VMX_EXIT_VMREAD:
7129	case VMX_EXIT_VMRESUME:
7130	case VMX_EXIT_VMWRITE:
7131	case VMX_EXIT_VMXOFF:
7132	case VMX_EXIT_VMXON:
7133	case VMX_EXIT_INVEPT:
7134	case VMX_EXIT_INVVPID:
7135	case VMX_EXIT_VMFUNC:
7136	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7137	break;
7138	default:
7139	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7140	break;
7141	}
7142	return rc;
7143	}
7144	#endif
7145
7146	#ifdef DEBUG
7147	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7148	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7149	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7150
7151	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7152	do \
7153	{ \
7154	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7155	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7156	} while (0)
7157
7158	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7159	do { \
7160	AssertPtr(pVCpu); \
7161	AssertPtr(pMixedCtx); \
7162	AssertPtr(pVmxTransient); \
7163	Assert(pVmxTransient->fVMEntryFailed == false); \
7164	Assert(ASMIntAreEnabled()); \
7165	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7166	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7167	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7168	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7169	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7170	HMVMX_ASSERT_PREEMPT_CPUID(); \
7171	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7172	} while (0)
7173
7174	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7175	do { \
7176	Log4Func(("\n")); \
7177	} while(0)
7178	#else /* Release builds */
7179	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7180	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7181	#endif
7182
7183
7184	/**
7185	* Advances the guest RIP after reading it from the VMCS.
7186	*
7187	* @returns VBox status code.
7188	* @param pVCpu Pointer to the VMCPU.
7189	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7190	* out-of-sync. Make sure to update the required fields
7191	* before using them.
7192	* @param pVmxTransient Pointer to the VMX transient structure.
7193	*
7194	* @remarks No-long-jump zone!!!
7195	*/
7196	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7197	{
7198	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7199	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7200	AssertRCReturn(rc, rc);
7201
7202	pMixedCtx->rip += pVmxTransient->cbInstr;
7203	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7204	return rc;
7205	}
7206
7207
7208	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7209	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7210	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7211	/**
7212	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7213	*/
7214	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7215	{
7216	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7217	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7218	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7219	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7220	Assert(ASMIntAreEnabled());
7221	return VINF_SUCCESS;
7222	#else
7223	return VINF_EM_RAW_INTERRUPT;
7224	#endif
7225	}
7226
7227
7228	/**
7229	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7230	*/
7231	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7232	{
7233	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7234	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7235
7236	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7237	AssertRCReturn(rc, rc);
7238
7239	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7240	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7241	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7242
7243	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7244	{
7245	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7246	return VINF_EM_RAW_INTERRUPT;
7247	}
7248
7249	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7250	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7251	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
7252	{
7253	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7254	return VINF_SUCCESS;
7255	}
7256	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7257	{
7258	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7259	return rc;
7260	}
7261
7262	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7263	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7264	switch (uIntrType)
7265	{
7266	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7267	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7268	/* no break */
7269	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7270	{
7271	switch (uVector)
7272	{
7273	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7274	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7275	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7276	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7277	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7278	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7279	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7280	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7281	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7282	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7283	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7284	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7285	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7286	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7287	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7288	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7289	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7290	#endif
7291	default:
7292	{
7293	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7294	AssertRCReturn(rc, rc);
7295
7296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7297	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7298	{
7299	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7300	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7301	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7302	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7303	AssertRCReturn(rc, rc);
7304	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7305	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7306	0 /* GCPtrFaultAddress */);
7307	AssertRCReturn(rc, rc);
7308	}
7309	else
7310	{
7311	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7312	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7313	}
7314	break;
7315	}
7316	}
7317	break;
7318	}
7319
7320	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7321	default:
7322	{
7323	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7324	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7325	break;
7326	}
7327	}
7328	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7329	return rc;
7330	}
7331
7332
7333	/**
7334	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7335	*/
7336	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7337	{
7338	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7339
7340	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7341	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7342	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7343	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7344	AssertRCReturn(rc, rc);
7345
7346	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7347	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7348	return VINF_SUCCESS;
7349	}
7350
7351
7352	/**
7353	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7354	*/
7355	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7356	{
7357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7358	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7359	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7360	}
7361
7362
7363	/**
7364	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7365	*/
7366	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7367	{
7368	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7370	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7371	}
7372
7373
7374	/**
7375	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7376	*/
7377	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7378	{
7379	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7381	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7382	}
7383
7384
7385	/**
7386	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7387	*/
7388	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7389	{
7390	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7391	PVM pVM = pVCpu->CTX_SUFF(pVM);
7392	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7393	if (RT_LIKELY(rc == VINF_SUCCESS))
7394	{
7395	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7396	Assert(pVmxTransient->cbInstr == 2);
7397	}
7398	else
7399	{
7400	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7401	rc = VERR_EM_INTERPRETER;
7402	}
7403	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7404	return rc;
7405	}
7406
7407
7408	/**
7409	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7410	*/
7411	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7412	{
7413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7414	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7415	AssertRCReturn(rc, rc);
7416
7417	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7418	return VINF_EM_RAW_EMULATE_INSTR;
7419
7420	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7421	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7422	}
7423
7424
7425	/**
7426	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7427	*/
7428	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7429	{
7430	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7431	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7432	AssertRCReturn(rc, rc);
7433
7434	PVM pVM = pVCpu->CTX_SUFF(pVM);
7435	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7436	if (RT_LIKELY(rc == VINF_SUCCESS))
7437	{
7438	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7439	Assert(pVmxTransient->cbInstr == 2);
7440	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7441	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7442	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7443	}
7444	else
7445	{
7446	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7447	rc = VERR_EM_INTERPRETER;
7448	}
7449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7450	return rc;
7451	}
7452
7453
7454	/**
7455	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7456	*/
7457	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7458	{
7459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7460	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7461	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7462	AssertRCReturn(rc, rc);
7463
7464	PVM pVM = pVCpu->CTX_SUFF(pVM);
7465	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7466	if (RT_LIKELY(rc == VINF_SUCCESS))
7467	{
7468	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7469	Assert(pVmxTransient->cbInstr == 3);
7470	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7471	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7472	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7473	}
7474	else
7475	{
7476	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7477	rc = VERR_EM_INTERPRETER;
7478	}
7479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7480	return rc;
7481	}
7482
7483
7484	/**
7485	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7486	*/
7487	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7488	{
7489	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7490	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7491	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7492	AssertRCReturn(rc, rc);
7493
7494	PVM pVM = pVCpu->CTX_SUFF(pVM);
7495	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7496	if (RT_LIKELY(rc == VINF_SUCCESS))
7497	{
7498	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7499	Assert(pVmxTransient->cbInstr == 2);
7500	}
7501	else
7502	{
7503	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7504	rc = VERR_EM_INTERPRETER;
7505	}
7506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7507	return rc;
7508	}
7509
7510
7511	/**
7512	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7513	*/
7514	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7515	{
7516	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7517	PVM pVM = pVCpu->CTX_SUFF(pVM);
7518	Assert(!pVM->hm.s.fNestedPaging);
7519
7520	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7521	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7522	AssertRCReturn(rc, rc);
7523
7524	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7525	rc = VBOXSTRICTRC_VAL(rc2);
7526	if (RT_LIKELY(rc == VINF_SUCCESS))
7527	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7528	else
7529	{
7530	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7531	pVmxTransient->uExitQualification, rc));
7532	}
7533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7534	return rc;
7535	}
7536
7537
7538	/**
7539	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7540	*/
7541	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7542	{
7543	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7544	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7545	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7546	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7547	AssertRCReturn(rc, rc);
7548
7549	PVM pVM = pVCpu->CTX_SUFF(pVM);
7550	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7551	if (RT_LIKELY(rc == VINF_SUCCESS))
7552	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7553	else
7554	{
7555	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7556	rc = VERR_EM_INTERPRETER;
7557	}
7558	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7559	return rc;
7560	}
7561
7562
7563	/**
7564	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7565	*/
7566	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7567	{
7568	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7569	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7570	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7571	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7572	AssertRCReturn(rc, rc);
7573
7574	PVM pVM = pVCpu->CTX_SUFF(pVM);
7575	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7576	rc = VBOXSTRICTRC_VAL(rc2);
7577	if (RT_LIKELY( rc == VINF_SUCCESS
7578	\|\| rc == VINF_EM_HALT))
7579	{
7580	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7581	AssertRCReturn(rc3, rc3);
7582
7583	if ( rc == VINF_EM_HALT
7584	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7585	{
7586	rc = VINF_SUCCESS;
7587	}
7588	}
7589	else
7590	{
7591	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7592	rc = VERR_EM_INTERPRETER;
7593	}
7594	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7595	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7597	return rc;
7598	}
7599
7600
7601	/**
7602	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7603	*/
7604	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7605	{
7606	/*
7607	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7608	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7609	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7610	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7611	*/
7612	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7613	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7614	}
7615
7616
7617	/**
7618	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7619	*/
7620	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7621	{
7622	/*
7623	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7624	* root operation. If we get there there is something funny going on.
7625	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7626	*/
7627	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7628	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7629	}
7630
7631
7632	/**
7633	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7634	*/
7635	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7636	{
7637	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7638	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7639	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7640	}
7641
7642
7643	/**
7644	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7645	*/
7646	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7647	{
7648	/*
7649	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7650	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7651	* See Intel spec. 25.3 "Other Causes of VM-exits".
7652	*/
7653	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7654	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7655	}
7656
7657
7658	/**
7659	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7660	* VM-exit.
7661	*/
7662	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7663	{
7664	/*
7665	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7666	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7667	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7668	*/
7669	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7670	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7671	}
7672
7673
7674	/**
7675	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7676	* VM-exit.
7677	*/
7678	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7679	{
7680	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7681	return VINF_EM_RESET;
7682	}
7683
7684
7685	/**
7686	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7687	*/
7688	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7689	{
7690	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7691	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7692	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7693	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7694	AssertRCReturn(rc, rc);
7695
7696	pMixedCtx->rip++;
7697	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7698	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7699	rc = VINF_SUCCESS;
7700	else
7701	rc = VINF_EM_HALT;
7702
7703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7704	return rc;
7705	}
7706
7707
7708	/**
7709	* VM-exit handler for instructions that result in a #UD exception delivered to
7710	* the guest.
7711	*/
7712	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7713	{
7714	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7715	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7716	return VINF_SUCCESS;
7717	}
7718
7719
7720	/**
7721	* VM-exit handler for expiry of the VMX preemption timer.
7722	*/
7723	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7724	{
7725	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7726
7727	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7728	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7729
7730	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7731	PVM pVM = pVCpu->CTX_SUFF(pVM);
7732	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7733	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7734	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7735	}
7736
7737
7738	/**
7739	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7740	*/
7741	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7742	{
7743	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7744
7745	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7746	/** @todo check if XSETBV is supported by the recompiler. */
7747	return VERR_EM_INTERPRETER;
7748	}
7749
7750
7751	/**
7752	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7753	*/
7754	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7755	{
7756	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7757
7758	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7759	/** @todo implement EMInterpretInvpcid() */
7760	return VERR_EM_INTERPRETER;
7761	}
7762
7763
7764	/**
7765	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7766	* Error VM-exit.
7767	*/
7768	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7769	{
7770	uint32_t uIntrState;
7771	HMVMXHCUINTREG uHCReg;
7772	uint64_t u64Val;
7773	uint32_t u32Val;
7774
7775	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7776	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7777	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7778	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7779	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7780	AssertRCReturn(rc, rc);
7781
7782	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7783	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7784	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7785	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7786
7787	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7788	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7789	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7790	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7791	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7792	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7793	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7794	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7795	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7796	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7797	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7798	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7799
7800	PVM pVM = pVCpu->CTX_SUFF(pVM);
7801	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7802
7803	return VERR_VMX_INVALID_GUEST_STATE;
7804	}
7805
7806
7807	/**
7808	* VM-exit handler for VM-entry failure due to an MSR-load
7809	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7810	*/
7811	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7812	{
7813	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7814	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7815	}
7816
7817
7818	/**
7819	* VM-exit handler for VM-entry failure due to a machine-check event
7820	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7821	*/
7822	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7823	{
7824	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7825	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7826	}
7827
7828
7829	/**
7830	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7831	* theory.
7832	*/
7833	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7834	{
7835	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7836	return VERR_VMX_UNDEFINED_EXIT_CODE;
7837	}
7838
7839
7840	/**
7841	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7842	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7843	* Conditional VM-exit.
7844	*/
7845	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7846	{
7847	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7848
7849	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7851	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7852	return VERR_EM_INTERPRETER;
7853	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7854	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7855	}
7856
7857
7858	/**
7859	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7860	*/
7861	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7862	{
7863	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7864
7865	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7867	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7868	return VERR_EM_INTERPRETER;
7869	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7870	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7871	}
7872
7873
7874	/**
7875	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7876	*/
7877	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7878	{
7879	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7880
7881	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7882	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7883	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7884	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7885	AssertRCReturn(rc, rc);
7886
7887	PVM pVM = pVCpu->CTX_SUFF(pVM);
7888	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7889	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7890	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7891	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7892
7893	if (RT_LIKELY(rc == VINF_SUCCESS))
7894	{
7895	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7896	Assert(pVmxTransient->cbInstr == 2);
7897	}
7898	return rc;
7899	}
7900
7901
7902	/**
7903	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7904	*/
7905	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7906	{
7907	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7908	PVM pVM = pVCpu->CTX_SUFF(pVM);
7909	int rc = VINF_SUCCESS;
7910
7911	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7912	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7913	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7914	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7915	AssertRCReturn(rc, rc);
7916	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
7917
7918	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7919	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7921
7922	if (RT_LIKELY(rc == VINF_SUCCESS))
7923	{
7924	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7925
7926	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7927	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7928	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7929	{
7930	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7931	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7932	}
7933	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7934	{
7935	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7936	AssertRCReturn(rc, rc);
7937	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7938	}
7939	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7940	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7941
7942	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7943	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7944	{
7945	switch (pMixedCtx->ecx)
7946	{
7947	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7948	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7949	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7950	case MSR_K8_FS_BASE: /* no break */
7951	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7952	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7953	}
7954	}
7955	#ifdef VBOX_STRICT
7956	else
7957	{
7958	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7959	switch (pMixedCtx->ecx)
7960	{
7961	case MSR_IA32_SYSENTER_CS:
7962	case MSR_IA32_SYSENTER_EIP:
7963	case MSR_IA32_SYSENTER_ESP:
7964	case MSR_K8_FS_BASE:
7965	case MSR_K8_GS_BASE:
7966	{
7967	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7968	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7969	}
7970
7971	case MSR_K8_LSTAR:
7972	case MSR_K6_STAR:
7973	case MSR_K8_SF_MASK:
7974	case MSR_K8_TSC_AUX:
7975	case MSR_K8_KERNEL_GS_BASE:
7976	{
7977	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7978	pMixedCtx->ecx));
7979	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7980	}
7981	}
7982	}
7983	#endif /* VBOX_STRICT */
7984	}
7985	return rc;
7986	}
7987
7988
7989	/**
7990	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7991	*/
7992	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7993	{
7994	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7995
7996	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7997	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7998	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7999	return VERR_EM_INTERPRETER;
8000	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8001	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8002	}
8003
8004
8005	/**
8006	* VM-exit handler for when the TPR value is lowered below the specified
8007	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8008	*/
8009	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8010	{
8011	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8012	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8013
8014	/*
8015	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8016	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8017	* resume guest execution.
8018	*/
8019	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8020	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8021	return VINF_SUCCESS;
8022	}
8023
8024
8025	/**
8026	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8027	* VM-exit.
8028	*
8029	* @retval VINF_SUCCESS when guest execution can continue.
8030	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8031	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8032	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8033	* recompiler.
8034	*/
8035	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8036	{
8037	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8038	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8039	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8040	AssertRCReturn(rc, rc);
8041
8042	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8043	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8044	PVM pVM = pVCpu->CTX_SUFF(pVM);
8045	switch (uAccessType)
8046	{
8047	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8048	{
8049	#if 0
8050	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8051	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8052	#else
8053	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8054	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8055	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8056	#endif
8057	AssertRCReturn(rc, rc);
8058
8059	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8060	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8061	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8062	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8063
8064	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8065	{
8066	case 0: /* CR0 */
8067	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8068	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8069	break;
8070	case 2: /* C2 **/
8071	/* Nothing to do here, CR2 it's not part of the VMCS. */
8072	break;
8073	case 3: /* CR3 */
8074	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8075	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8076	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8077	break;
8078	case 4: /* CR4 */
8079	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8080	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8081	break;
8082	case 8: /* CR8 */
8083	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8084	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8085	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8086	break;
8087	default:
8088	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8089	break;
8090	}
8091
8092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8093	break;
8094	}
8095
8096	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8097	{
8098	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8099	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8100	AssertRCReturn(rc, rc);
8101	Assert( !pVM->hm.s.fNestedPaging
8102	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8103	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8104
8105	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8106	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8107	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8108
8109	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8110	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8111	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8112	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8113	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8114	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8115	break;
8116	}
8117
8118	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8119	{
8120	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8121	AssertRCReturn(rc, rc);
8122	rc = EMInterpretCLTS(pVM, pVCpu);
8123	AssertRCReturn(rc, rc);
8124	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8126	Log4(("CRX CLTS write rc=%d\n", rc));
8127	break;
8128	}
8129
8130	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8131	{
8132	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8133	AssertRCReturn(rc, rc);
8134	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8135	if (RT_LIKELY(rc == VINF_SUCCESS))
8136	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8137	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8138	Log4(("CRX LMSW write rc=%d\n", rc));
8139	break;
8140	}
8141
8142	default:
8143	{
8144	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8145	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8146	}
8147	}
8148
8149	/* Validate possible error codes. */
8150	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8151	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8152	if (RT_SUCCESS(rc))
8153	{
8154	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8155	AssertRCReturn(rc2, rc2);
8156	}
8157
8158	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8159	return rc;
8160	}
8161
8162
8163	/**
8164	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8165	* VM-exit.
8166	*/
8167	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8168	{
8169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8170	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8171
8172	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8173	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8174	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8175	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8176	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8177	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8178	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8179	AssertRCReturn(rc, rc);
8180
8181	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8182
8183	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8184	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8185	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8186	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8187	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8188	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8189	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8190
8191	/* I/O operation lookup arrays. */
8192	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
8193	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8194
8195	const uint32_t cbSize = s_aIOSize[uIOWidth];
8196	const uint32_t cbInstr = pVmxTransient->cbInstr;
8197	PVM pVM = pVCpu->CTX_SUFF(pVM);
8198	if (fIOString)
8199	{
8200	/* INS/OUTS - I/O String instruction. */
8201	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8202	/** @todo for now manually disassemble later optimize by getting the fields from
8203	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8204	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8205	* segment prefix info. */
8206	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8207	if (RT_SUCCESS(rc))
8208	{
8209	if (fIOWrite)
8210	{
8211	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8212	(DISCPUMODE)pDis->uAddrMode, cbSize);
8213	rc = VBOXSTRICTRC_VAL(rc2);
8214	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8215	}
8216	else
8217	{
8218	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8219	(DISCPUMODE)pDis->uAddrMode, cbSize);
8220	rc = VBOXSTRICTRC_VAL(rc2);
8221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8222	}
8223	}
8224	else
8225	{
8226	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8227	rc = VINF_EM_RAW_EMULATE_INSTR;
8228	}
8229	}
8230	else
8231	{
8232	/* IN/OUT - I/O instruction. */
8233	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8234	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8235	if (fIOWrite)
8236	{
8237	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8238	rc = VBOXSTRICTRC_VAL(rc2);
8239	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8240	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8242	}
8243	else
8244	{
8245	uint32_t u32Result = 0;
8246	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8247	rc = VBOXSTRICTRC_VAL(rc2);
8248	if (IOM_SUCCESS(rc))
8249	{
8250	/* Save result of I/O IN instr. in AL/AX/EAX. */
8251	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8252	}
8253	else if (rc == VINF_IOM_R3_IOPORT_READ)
8254	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8256	}
8257	}
8258
8259	if (IOM_SUCCESS(rc))
8260	{
8261	pMixedCtx->rip += cbInstr;
8262	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8263	if (RT_LIKELY(rc == VINF_SUCCESS))
8264	{
8265	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8266	AssertRCReturn(rc, rc);
8267
8268	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8269	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8270	{
8271	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8272	for (unsigned i = 0; i < 4; i++)
8273	{
8274	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8275	if ( ( uIOPort >= pMixedCtx->dr[i]
8276	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8277	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8278	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8279	{
8280	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8281	uint64_t uDR6 = ASMGetDR6();
8282
8283	/* Clear all breakpoint status flags and set the one we just hit. */
8284	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8285	uDR6 \|= (uint64_t)RT_BIT(i);
8286
8287	/*
8288	* Note: AMD64 Architecture Programmer's Manual 13.1:
8289	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8290	* be cleared by software after the contents have been read.
8291	*/
8292	ASMSetDR6(uDR6);
8293
8294	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8295	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8296
8297	/* Paranoia. */
8298	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8299	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8300	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8301
8302	/* Resync DR7 */
8303	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8304	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8305
8306	/* Set #DB to be injected into the VM and continue guest execution. */
8307	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8308	break;
8309	}
8310	}
8311	}
8312	}
8313	}
8314
8315	#ifdef DEBUG
8316	if (rc == VINF_IOM_R3_IOPORT_READ)
8317	Assert(!fIOWrite);
8318	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8319	Assert(fIOWrite);
8320	else
8321	{
8322	AssertMsg( RT_FAILURE(rc)
8323	\|\| rc == VINF_SUCCESS
8324	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8325	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8326	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8327	}
8328	#endif
8329
8330	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8331	return rc;
8332	}
8333
8334
8335	/**
8336	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8337	* VM-exit.
8338	*/
8339	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8340	{
8341	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8342
8343	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8344	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8345	AssertRCReturn(rc, rc);
8346	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8347	{
8348	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8349	AssertRCReturn(rc, rc);
8350	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8351	{
8352	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8353
8354	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8355	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8356	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8357	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8358	{
8359	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8360	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8361
8362	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8363	Assert(!pVCpu->hm.s.Event.fPending);
8364	pVCpu->hm.s.Event.fPending = true;
8365	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8366	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8367	AssertRCReturn(rc, rc);
8368	if (fErrorCodeValid)
8369	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8370	else
8371	pVCpu->hm.s.Event.u32ErrCode = 0;
8372	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8373	&& uVector == X86_XCPT_PF)
8374	{
8375	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8376	}
8377
8378	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8379	}
8380	}
8381	}
8382
8383	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8384	* emulation. */
8385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8386	return VERR_EM_INTERPRETER;
8387	}
8388
8389
8390	/**
8391	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8392	*/
8393	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8394	{
8395	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8396	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8397	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8398	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8399	AssertRCReturn(rc, rc);
8400	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8401	return VINF_EM_DBG_STOP;
8402	}
8403
8404
8405	/**
8406	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8407	*/
8408	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8409	{
8410	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8411
8412	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8413	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8414	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8415	return VINF_SUCCESS;
8416	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8417	return rc;
8418
8419	#if 0
8420	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8421	* just sync the whole thing. */
8422	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8423	#else
8424	/* Aggressive state sync. for now. */
8425	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8426	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8427	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8428	#endif
8429	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8430	AssertRCReturn(rc, rc);
8431
8432	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8433	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8434	switch (uAccessType)
8435	{
8436	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8437	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8438	{
8439	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8440	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8441	{
8442	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8443	}
8444
8445	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8446	GCPhys &= PAGE_BASE_GC_MASK;
8447	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8448	PVM pVM = pVCpu->CTX_SUFF(pVM);
8449	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8450	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8451
8452	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8453	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8454	CPUMCTX2CORE(pMixedCtx), GCPhys);
8455	rc = VBOXSTRICTRC_VAL(rc2);
8456	Log4(("ApicAccess rc=%d\n", rc));
8457	if ( rc == VINF_SUCCESS
8458	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8459	\|\| rc == VERR_PAGE_NOT_PRESENT)
8460	{
8461	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8462	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8463	rc = VINF_SUCCESS;
8464	}
8465	break;
8466	}
8467
8468	default:
8469	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8470	rc = VINF_EM_RAW_EMULATE_INSTR;
8471	break;
8472	}
8473
8474	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8475	return rc;
8476	}
8477
8478
8479	/**
8480	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8481	* VM-exit.
8482	*/
8483	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8484	{
8485	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8486
8487	/* We should -not- get this VM-exit if the guest is debugging. */
8488	if (CPUMIsGuestDebugStateActive(pVCpu))
8489	{
8490	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8491	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8492	}
8493
8494	int rc = VERR_INTERNAL_ERROR_5;
8495	if ( !DBGFIsStepping(pVCpu)
8496	&& !CPUMIsHyperDebugStateActive(pVCpu))
8497	{
8498	/* Don't intercept MOV DRx. */
8499	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8500	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8501	AssertRCReturn(rc, rc);
8502
8503	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8504	PVM pVM = pVCpu->CTX_SUFF(pVM);
8505	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8506	AssertRC(rc);
8507	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8508
8509	#ifdef VBOX_WITH_STATISTICS
8510	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8511	AssertRCReturn(rc, rc);
8512	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8514	else
8515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8516	#endif
8517	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8518	return VINF_SUCCESS;
8519	}
8520
8521	/*
8522	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8523	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8524	*/
8525	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8526	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8527	AssertRCReturn(rc, rc);
8528
8529	PVM pVM = pVCpu->CTX_SUFF(pVM);
8530	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8531	{
8532	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8533	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8534	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8535	if (RT_SUCCESS(rc))
8536	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8537	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8538	}
8539	else
8540	{
8541	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8542	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8543	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8544	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8545	}
8546
8547	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8548	if (RT_SUCCESS(rc))
8549	{
8550	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8551	AssertRCReturn(rc2, rc2);
8552	}
8553	return rc;
8554	}
8555
8556
8557	/**
8558	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8559	* Conditional VM-exit.
8560	*/
8561	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8562	{
8563	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8564	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8565
8566	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8567	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8568	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8569	return VINF_SUCCESS;
8570	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8571	return rc;
8572
8573	RTGCPHYS GCPhys = 0;
8574	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8575
8576	#if 0
8577	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8578	#else
8579	/* Aggressive state sync. for now. */
8580	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8581	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8582	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8583	#endif
8584	AssertRCReturn(rc, rc);
8585
8586	/*
8587	* If we succeed, resume guest execution.
8588	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8589	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8590	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8591	* weird case. See @bugref{6043}.
8592	*/
8593	PVM pVM = pVCpu->CTX_SUFF(pVM);
8594	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8595	rc = VBOXSTRICTRC_VAL(rc2);
8596	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8597	if ( rc == VINF_SUCCESS
8598	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8599	\|\| rc == VERR_PAGE_NOT_PRESENT)
8600	{
8601	/* Successfully handled MMIO operation. */
8602	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8603	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8604	rc = VINF_SUCCESS;
8605	}
8606	return rc;
8607	}
8608
8609
8610	/**
8611	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8612	* VM-exit.
8613	*/
8614	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8615	{
8616	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8617	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8618
8619	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8620	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8621	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8622	return VINF_SUCCESS;
8623	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8624	return rc;
8625
8626	RTGCPHYS GCPhys = 0;
8627	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8628	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8629	#if 0
8630	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8631	#else
8632	/* Aggressive state sync. for now. */
8633	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8634	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8635	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8636	#endif
8637	AssertRCReturn(rc, rc);
8638
8639	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8640	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8641
8642	RTGCUINT uErrorCode = 0;
8643	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8644	uErrorCode \|= X86_TRAP_PF_ID;
8645	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8646	uErrorCode \|= X86_TRAP_PF_RW;
8647	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8648	uErrorCode \|= X86_TRAP_PF_P;
8649
8650	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8651
8652	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8653	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8654
8655	/* Handle the pagefault trap for the nested shadow table. */
8656	PVM pVM = pVCpu->CTX_SUFF(pVM);
8657	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8658	TRPMResetTrap(pVCpu);
8659
8660	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8661	if ( rc == VINF_SUCCESS
8662	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8663	\|\| rc == VERR_PAGE_NOT_PRESENT)
8664	{
8665	/* Successfully synced our nested page tables. */
8666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8667	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8668	return VINF_SUCCESS;
8669	}
8670
8671	Log4(("EPT return to ring-3 rc=%d\n"));
8672	return rc;
8673	}
8674
8675
8676	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8677	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8678	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8679	/**
8680	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8681	*/
8682	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8683	{
8684	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8686
8687	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8688	AssertRCReturn(rc, rc);
8689
8690	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8691	{
8692	/* Old-style FPU error reporting needs some extra work. */
8693	/** @todo don't fall back to the recompiler, but do it manually. */
8694	return VERR_EM_INTERPRETER;
8695	}
8696	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8697	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8698	return rc;
8699	}
8700
8701
8702	/**
8703	* VM-exit exception handler for #BP (Breakpoint exception).
8704	*/
8705	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8706	{
8707	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8709
8710	/** @todo Try optimize this by not saving the entire guest state unless
8711	* really needed. */
8712	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8713	AssertRCReturn(rc, rc);
8714
8715	PVM pVM = pVCpu->CTX_SUFF(pVM);
8716	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8717	if (rc == VINF_EM_RAW_GUEST_TRAP)
8718	{
8719	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8720	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8721	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8722	AssertRCReturn(rc, rc);
8723
8724	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8725	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8726	}
8727
8728	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8729	return rc;
8730	}
8731
8732
8733	/**
8734	* VM-exit exception handler for #DB (Debug exception).
8735	*/
8736	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8737	{
8738	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8739	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8740
8741	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8742	AssertRCReturn(rc, rc);
8743
8744	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8745	uint64_t uDR6 = X86_DR6_INIT_VAL;
8746	uDR6 \|= (pVmxTransient->uExitQualification
8747	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8748	PVM pVM = pVCpu->CTX_SUFF(pVM);
8749	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8750	if (rc == VINF_EM_RAW_GUEST_TRAP)
8751	{
8752	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8753	pMixedCtx->dr[6] = uDR6;
8754
8755	if (CPUMIsGuestDebugStateActive(pVCpu))
8756	ASMSetDR6(pMixedCtx->dr[6]);
8757
8758	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8759
8760	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8761	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8762
8763	/* Paranoia. */
8764	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8765	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8766	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8767
8768	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8769	AssertRCReturn(rc,rc);
8770
8771	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8772	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8773	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8774	AssertRCReturn(rc2, rc2);
8775	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8776	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8777	rc = VINF_SUCCESS;
8778	}
8779
8780	return rc;
8781	}
8782
8783
8784	/**
8785	* VM-exit exception handler for #NM (Device-not-available exception: floating
8786	* point exception).
8787	*/
8788	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8789	{
8790	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8791
8792	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8793	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8794	#endif
8795
8796	/* We require CR0 and EFER. EFER is always up-to-date. */
8797	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8798	AssertRCReturn(rc, rc);
8799
8800	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8801	PVM pVM = pVCpu->CTX_SUFF(pVM);
8802	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8803	if (rc == VINF_SUCCESS)
8804	{
8805	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8806	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8808	return VINF_SUCCESS;
8809	}
8810
8811	/* Forward #NM to the guest. */
8812	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8813	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8814	AssertRCReturn(rc, rc);
8815	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8816	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8818	return rc;
8819	}
8820
8821
8822	/**
8823	* VM-exit exception handler for #GP (General-protection exception).
8824	*
8825	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8826	*/
8827	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8828	{
8829	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8831
8832	int rc = VERR_INTERNAL_ERROR_5;
8833	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8834	{
8835	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8836	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8837	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8838	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8839	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8840	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8841	AssertRCReturn(rc, rc);
8842	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8843	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8844	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8845	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8846	return rc;
8847	#else
8848	/* We don't intercept #GP. */
8849	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8850	return VERR_VMX_UNEXPECTED_EXCEPTION;
8851	#endif
8852	}
8853
8854	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8855	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8856
8857	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8858	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8859	AssertRCReturn(rc, rc);
8860
8861	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8862	uint32_t cbOp = 0;
8863	PVM pVM = pVCpu->CTX_SUFF(pVM);
8864	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8865	if (RT_SUCCESS(rc))
8866	{
8867	rc = VINF_SUCCESS;
8868	Assert(cbOp == pDis->cbInstr);
8869	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8870	switch (pDis->pCurInstr->uOpcode)
8871	{
8872	case OP_CLI:
8873	{
8874	pMixedCtx->eflags.Bits.u1IF = 0;
8875	pMixedCtx->rip += pDis->cbInstr;
8876	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8877	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8878	break;
8879	}
8880
8881	case OP_STI:
8882	{
8883	pMixedCtx->eflags.Bits.u1IF = 1;
8884	pMixedCtx->rip += pDis->cbInstr;
8885	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8886	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8887	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8889	break;
8890	}
8891
8892	case OP_HLT:
8893	{
8894	rc = VINF_EM_HALT;
8895	pMixedCtx->rip += pDis->cbInstr;
8896	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8897	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8898	break;
8899	}
8900
8901	case OP_POPF:
8902	{
8903	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8904	uint32_t cbParm = 0;
8905	uint32_t uMask = 0;
8906	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8907	{
8908	cbParm = 4;
8909	uMask = 0xffffffff;
8910	}
8911	else
8912	{
8913	cbParm = 2;
8914	uMask = 0xffff;
8915	}
8916
8917	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8918	RTGCPTR GCPtrStack = 0;
8919	X86EFLAGS uEflags;
8920	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8921	&GCPtrStack);
8922	if (RT_SUCCESS(rc))
8923	{
8924	Assert(sizeof(uEflags.u32) >= cbParm);
8925	uEflags.u32 = 0;
8926	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8927	}
8928	if (RT_FAILURE(rc))
8929	{
8930	rc = VERR_EM_INTERPRETER;
8931	break;
8932	}
8933	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8934	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8935	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8936	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8937	pMixedCtx->eflags.Bits.u1RF = 0;
8938	pMixedCtx->esp += cbParm;
8939	pMixedCtx->esp &= uMask;
8940	pMixedCtx->rip += pDis->cbInstr;
8941	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8943	break;
8944	}
8945
8946	case OP_PUSHF:
8947	{
8948	uint32_t cbParm = 0;
8949	uint32_t uMask = 0;
8950	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8951	{
8952	cbParm = 4;
8953	uMask = 0xffffffff;
8954	}
8955	else
8956	{
8957	cbParm = 2;
8958	uMask = 0xffff;
8959	}
8960
8961	/* Get the stack pointer & push the contents of eflags onto the stack. */
8962	RTGCPTR GCPtrStack = 0;
8963	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8964	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8965	if (RT_FAILURE(rc))
8966	{
8967	rc = VERR_EM_INTERPRETER;
8968	break;
8969	}
8970	X86EFLAGS uEflags;
8971	uEflags = pMixedCtx->eflags;
8972	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8973	uEflags.Bits.u1RF = 0;
8974	uEflags.Bits.u1VM = 0;
8975
8976	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8977	if (RT_FAILURE(rc))
8978	{
8979	rc = VERR_EM_INTERPRETER;
8980	break;
8981	}
8982	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8983	pMixedCtx->esp -= cbParm;
8984	pMixedCtx->esp &= uMask;
8985	pMixedCtx->rip += pDis->cbInstr;
8986	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8988	break;
8989	}
8990
8991	case OP_IRET:
8992	{
8993	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8994	* instruction reference. */
8995	RTGCPTR GCPtrStack = 0;
8996	uint32_t uMask = 0xffff;
8997	uint16_t aIretFrame[3];
8998	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8999	{
9000	rc = VERR_EM_INTERPRETER;
9001	break;
9002	}
9003	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9004	&GCPtrStack);
9005	if (RT_SUCCESS(rc))
9006	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9007	if (RT_FAILURE(rc))
9008	{
9009	rc = VERR_EM_INTERPRETER;
9010	break;
9011	}
9012	pMixedCtx->eip = 0;
9013	pMixedCtx->ip = aIretFrame[0];
9014	pMixedCtx->cs.Sel = aIretFrame[1];
9015	pMixedCtx->cs.ValidSel = aIretFrame[1];
9016	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9017	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9018	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9019	pMixedCtx->sp += sizeof(aIretFrame);
9020	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9021	\| HM_CHANGED_GUEST_RFLAGS;
9022	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9023	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9024	break;
9025	}
9026
9027	case OP_INT:
9028	{
9029	uint16_t uVector = pDis->Param1.uValue & 0xff;
9030	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9031	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9032	break;
9033	}
9034
9035	case OP_INTO:
9036	{
9037	if (pMixedCtx->eflags.Bits.u1OF)
9038	{
9039	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9040	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9041	}
9042	break;
9043	}
9044
9045	default:
9046	{
9047	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9048	EMCODETYPE_SUPERVISOR);
9049	rc = VBOXSTRICTRC_VAL(rc2);
9050	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9051	Log4(("#GP rc=%Rrc\n", rc));
9052	break;
9053	}
9054	}
9055	}
9056	else
9057	rc = VERR_EM_INTERPRETER;
9058
9059	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9060	("#GP Unexpected rc=%Rrc\n", rc));
9061	return rc;
9062	}
9063
9064
9065	/**
9066	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9067	* the exception reported in the VMX transient structure back into the VM.
9068	*
9069	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9070	* up-to-date.
9071	*/
9072	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9073	{
9074	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9075
9076	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9077	hmR0VmxCheckExitDueToEventDelivery(). */
9078	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9079	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9080	AssertRCReturn(rc, rc);
9081	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9082	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9083	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9084	return VINF_SUCCESS;
9085	}
9086
9087
9088	/**
9089	* VM-exit exception handler for #PF (Page-fault exception).
9090	*/
9091	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9092	{
9093	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9094	PVM pVM = pVCpu->CTX_SUFF(pVM);
9095	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9096	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9097	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9098	AssertRCReturn(rc, rc);
9099
9100	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9101	if (pVM->hm.s.fNestedPaging)
9102	{
9103	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
9104	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9105	{
9106	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9107	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9108	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9109	}
9110	else
9111	{
9112	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9113	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9114	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9115	}
9116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9117	return rc;
9118	}
9119	#else
9120	Assert(!pVM->hm.s.fNestedPaging);
9121	#endif
9122
9123	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9124	AssertRCReturn(rc, rc);
9125
9126	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9127	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9128
9129	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9130	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9131	(RTGCPTR)pVmxTransient->uExitQualification);
9132
9133	Log4(("#PF: rc=%Rrc\n", rc));
9134	if (rc == VINF_SUCCESS)
9135	{
9136	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9137	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9138	* memory? We don't update the whole state here... */
9139	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9140	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9141	TRPMResetTrap(pVCpu);
9142	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9143	return rc;
9144	}
9145	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9146	{
9147	if (!pVmxTransient->fVectoringPF)
9148	{
9149	/* It's a guest page fault and needs to be reflected to the guest. */
9150	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9151	TRPMResetTrap(pVCpu);
9152	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9153	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9154	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9155	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9156	}
9157	else
9158	{
9159	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9160	TRPMResetTrap(pVCpu);
9161	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
9162	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9163	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9164	}
9165
9166	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9167	return VINF_SUCCESS;
9168	}
9169
9170	TRPMResetTrap(pVCpu);
9171	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9172	return rc;
9173	}
9174

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46664

Download in other formats: