HMVMXR0.cpp@ 46502

Last change on this file since 46502 was 46498, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Comments.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 387.4 KB

Line
1	/* $Id: HMVMXR0.cpp 46498 2013-06-11 15:54:28Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/**
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG uEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
174	uint64_t u64LStarMsr;
175	/** The guest's TPR value used for TPR shadowing. */
176	uint8_t u8GuestTpr;
177	/** Alignment. */
178	uint8_t abAlignment0[6];
179
180	/** The basic VM-exit reason. */
181	uint16_t uExitReason;
182	/** Alignment. */
183	uint16_t u16Alignment0;
184	/** The VM-exit interruption error code. */
185	uint32_t uExitIntrErrorCode;
186	/** The VM-exit exit qualification. */
187	uint64_t uExitQualification;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntrInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** Whether the VM-entry failed or not. */
194	bool fVMEntryFailed;
195	/** Alignment. */
196	uint8_t abAlignment1[5];
197
198	/** The VM-entry interruption-information field. */
199	uint32_t uEntryIntrInfo;
200	/** The VM-entry exception error code field. */
201	uint32_t uEntryXcptErrorCode;
202	/** The VM-entry instruction length field. */
203	uint32_t cbEntryInstr;
204
205	/** IDT-vectoring information field. */
206	uint32_t uIdtVectoringInfo;
207	/** IDT-vectoring error code. */
208	uint32_t uIdtVectoringErrorCode;
209
210	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
211	uint32_t fVmcsFieldsRead;
212	/** Whether TSC-offsetting should be setup before VM-entry. */
213	bool fUpdateTscOffsettingAndPreemptTimer;
214	/** Whether the VM-exit was caused by a page-fault during delivery of a
215	* contributary exception or a page-fault. */
216	bool fVectoringPF;
217	} VMXTRANSIENT, *PVMXTRANSIENT;
218	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
219	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
220	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
221
222
223	/**
224	* MSR-bitmap read permissions.
225	*/
226	typedef enum VMXMSREXITREAD
227	{
228	/** Reading this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_READ = 0xb,
230	/** Reading this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_READ
232	} VMXMSREXITREAD;
233
234	/**
235	* MSR-bitmap write permissions.
236	*/
237	typedef enum VMXMSREXITWRITE
238	{
239	/** Writing to this MSR causes a VM-exit. */
240	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
241	/** Writing to this MSR does not cause a VM-exit. */
242	VMXMSREXIT_PASSTHRU_WRITE
243	} VMXMSREXITWRITE;
244
245
246	/*******************************************************************************
247	* Internal Functions *
248	*******************************************************************************/
249	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
250	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
251	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
252	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
253	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
254	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
255	#endif
256	#ifndef HMVMX_USE_FUNCTION_TABLE
257	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
258	#define HMVMX_EXIT_DECL static int
259	#else
260	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
261	#endif
262
263	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307
308	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315
316
317	/*******************************************************************************
318	* Global Variables *
319	*******************************************************************************/
320	#ifdef HMVMX_USE_FUNCTION_TABLE
321	/**
322	* VM-exit handler.
323	*
324	* @returns VBox status code.
325	* @param pVCpu Pointer to the VMCPU.
326	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
327	* out-of-sync. Make sure to update the required
328	* fields before using them.
329	* @param pVmxTransient Pointer to the VMX-transient structure.
330	*/
331	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332	/** Pointer to VM-exit handler. */
333	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
334
335	/**
336	* VMX_EXIT dispatch table.
337	*/
338	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
339	{
340	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
341	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
342	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
343	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
344	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
345	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
346	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
347	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
348	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
349	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
350	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
351	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
352	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
353	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
354	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
355	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
356	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
357	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
358	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
359	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
360	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
361	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
362	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
363	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
364	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
365	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
366	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
367	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
368	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
369	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
370	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
371	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
372	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
373	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
374	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
375	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
376	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
377	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
378	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
379	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
380	/* 40 UNDEFINED */ hmR0VmxExitPause,
381	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
382	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
383	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
384	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
385	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
387	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
388	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
389	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
390	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
391	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
392	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
393	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
394	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
395	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
396	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
397	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
398	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
399	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
400	};
401	#endif /* HMVMX_USE_FUNCTION_TABLE */
402
403	#ifdef VBOX_STRICT
404	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
405	{
406	/* 0 */ "(Not Used)",
407	/* 1 */ "VMCALL executed in VMX root operation.",
408	/* 2 */ "VMCLEAR with invalid physical address.",
409	/* 3 */ "VMCLEAR with VMXON pointer.",
410	/* 4 */ "VMLAUNCH with non-clear VMCS.",
411	/* 5 */ "VMRESUME with non-launched VMCS.",
412	/* 6 */ "VMRESUME after VMXOFF",
413	/* 7 */ "VM entry with invalid control fields.",
414	/* 8 */ "VM entry with invalid host state fields.",
415	/* 9 */ "VMPTRLD with invalid physical address.",
416	/* 10 */ "VMPTRLD with VMXON pointer.",
417	/* 11 */ "VMPTRLD with incorrect revision identifier.",
418	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
419	/* 13 */ "VMWRITE to read-only VMCS component.",
420	/* 14 */ "(Not Used)",
421	/* 15 */ "VMXON executed in VMX root operation.",
422	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
423	/* 17 */ "VM entry with non-launched executing VMCS.",
424	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
425	/* 19 */ "VMCALL with non-clear VMCS.",
426	/* 20 */ "VMCALL with invalid VM-exit control fields.",
427	/* 21 */ "(Not Used)",
428	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
429	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
430	/* 24 */ "VMCALL with invalid SMM-monitor features.",
431	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
432	/* 26 */ "VM entry with events blocked by MOV SS.",
433	/* 27 */ "(Not Used)",
434	/* 28 */ "Invalid operand to INVEPT/INVVPID."
435	};
436	#endif /* VBOX_STRICT */
437
438
439
440	/**
441	* Updates the VM's last error record. If there was a VMX instruction error,
442	* reads the error data from the VMCS and updates VCPU's last error record as
443	* well.
444	*
445	* @param pVM Pointer to the VM.
446	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
447	* VERR_VMX_UNABLE_TO_START_VM or
448	* VERR_VMX_INVALID_VMCS_FIELD).
449	* @param rc The error code.
450	*/
451	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
452	{
453	AssertPtr(pVM);
454	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
455	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
456	{
457	AssertPtrReturnVoid(pVCpu);
458	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
459	}
460	pVM->hm.s.lLastError = rc;
461	}
462
463
464	/**
465	* Reads the VM-entry interruption-information field from the VMCS into the VMX
466	* transient structure.
467	*
468	* @returns VBox status code.
469	* @param pVmxTransient Pointer to the VMX transient structure.
470	*
471	* @remarks No-long-jump zone!!!
472	*/
473	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
474	{
475	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
476	AssertRCReturn(rc, rc);
477	return VINF_SUCCESS;
478	}
479
480
481	/**
482	* Reads the VM-entry exception error code field from the VMCS into
483	* the VMX transient structure.
484	*
485	* @returns VBox status code.
486	* @param pVmxTransient Pointer to the VMX transient structure.
487	*
488	* @remarks No-long-jump zone!!!
489	*/
490	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
491	{
492	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
493	AssertRCReturn(rc, rc);
494	return VINF_SUCCESS;
495	}
496
497
498	/**
499	* Reads the VM-entry exception error code field from the VMCS into
500	* the VMX transient structure.
501	*
502	* @returns VBox status code.
503	* @param pVCpu Pointer to the VMCPU.
504	* @param pVmxTransient Pointer to the VMX transient structure.
505	*
506	* @remarks No-long-jump zone!!!
507	*/
508	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
509	{
510	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
511	AssertRCReturn(rc, rc);
512	return VINF_SUCCESS;
513	}
514
515
516	/**
517	* Reads the VM-exit interruption-information field from the VMCS into the VMX
518	* transient structure.
519	*
520	* @returns VBox status code.
521	* @param pVCpu Pointer to the VMCPU.
522	* @param pVmxTransient Pointer to the VMX transient structure.
523	*/
524	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
525	{
526	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
527	{
528	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
529	AssertRCReturn(rc, rc);
530	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
531	}
532	return VINF_SUCCESS;
533	}
534
535
536	/**
537	* Reads the VM-exit interruption error code from the VMCS into the VMX
538	* transient structure.
539	*
540	* @returns VBox status code.
541	* @param pVCpu Pointer to the VMCPU.
542	* @param pVmxTransient Pointer to the VMX transient structure.
543	*/
544	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
545	{
546	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
547	{
548	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
549	AssertRCReturn(rc, rc);
550	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
551	}
552	return VINF_SUCCESS;
553	}
554
555
556	/**
557	* Reads the VM-exit instruction length field from the VMCS into the VMX
558	* transient structure.
559	*
560	* @returns VBox status code.
561	* @param pVCpu Pointer to the VMCPU.
562	* @param pVmxTransient Pointer to the VMX transient structure.
563	*/
564	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
565	{
566	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
567	{
568	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
569	AssertRCReturn(rc, rc);
570	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
571	}
572	return VINF_SUCCESS;
573	}
574
575
576	/**
577	* Reads the exit qualification from the VMCS into the VMX transient structure.
578	*
579	* @returns VBox status code.
580	* @param pVCpu Pointer to the VMCPU.
581	* @param pVmxTransient Pointer to the VMX transient structure.
582	*/
583	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
584	{
585	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
586	{
587	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
588	AssertRCReturn(rc, rc);
589	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
590	}
591	return VINF_SUCCESS;
592	}
593
594
595	/**
596	* Reads the IDT-vectoring information field from the VMCS into the VMX
597	* transient structure.
598	*
599	* @returns VBox status code.
600	* @param pVmxTransient Pointer to the VMX transient structure.
601	*
602	* @remarks No-long-jump zone!!!
603	*/
604	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
605	{
606	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
607	{
608	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
609	AssertRCReturn(rc, rc);
610	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
611	}
612	return VINF_SUCCESS;
613	}
614
615
616	/**
617	* Reads the IDT-vectoring error code from the VMCS into the VMX
618	* transient structure.
619	*
620	* @returns VBox status code.
621	* @param pVmxTransient Pointer to the VMX transient structure.
622	*/
623	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
624	{
625	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
626	{
627	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
628	AssertRCReturn(rc, rc);
629	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
630	}
631	return VINF_SUCCESS;
632	}
633
634
635	/**
636	* Enters VMX root mode operation on the current CPU.
637	*
638	* @returns VBox status code.
639	* @param pVM Pointer to the VM (optional, can be NULL, after
640	* a resume).
641	* @param HCPhysCpuPage Physical address of the VMXON region.
642	* @param pvCpuPage Pointer to the VMXON region.
643	*/
644	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
645	{
646	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
647	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
648	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
649
650	if (pVM)
651	{
652	/* Write the VMCS revision dword to the VMXON region. */
653	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
654	}
655
656	/* Enable the VMX bit in CR4 if necessary. */
657	RTCCUINTREG uCr4 = ASMGetCR4();
658	if (!(uCr4 & X86_CR4_VMXE))
659	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
660
661	/* Enter VMX root mode. */
662	int rc = VMXEnable(HCPhysCpuPage);
663	if (RT_FAILURE(rc))
664	ASMSetCR4(uCr4);
665
666	return rc;
667	}
668
669
670	/**
671	* Exits VMX root mode operation on the current CPU.
672	*
673	* @returns VBox status code.
674	*/
675	static int hmR0VmxLeaveRootMode(void)
676	{
677	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
678
679	/* If we're for some reason not in VMX root mode, then don't leave it. */
680	RTCCUINTREG uHostCR4 = ASMGetCR4();
681	if (uHostCR4 & X86_CR4_VMXE)
682	{
683	/* Exit VMX root mode and clear the VMX bit in CR4. */
684	VMXDisable();
685	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
686	return VINF_SUCCESS;
687	}
688
689	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
690	}
691
692
693	/**
694	* Allocates and maps one physically contiguous page. The allocated page is
695	* zero'd out. (Used by various VT-x structures).
696	*
697	* @returns IPRT status code.
698	* @param pMemObj Pointer to the ring-0 memory object.
699	* @param ppVirt Where to store the virtual address of the
700	* allocation.
701	* @param pPhys Where to store the physical address of the
702	* allocation.
703	*/
704	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
705	{
706	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
707	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
708	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
709
710	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
711	if (RT_FAILURE(rc))
712	return rc;
713	ppVirt = RTR0MemObjAddress(pMemObj);
714	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
715	ASMMemZero32(*ppVirt, PAGE_SIZE);
716	return VINF_SUCCESS;
717	}
718
719
720	/**
721	* Frees and unmaps an allocated physical page.
722	*
723	* @param pMemObj Pointer to the ring-0 memory object.
724	* @param ppVirt Where to re-initialize the virtual address of
725	* allocation as 0.
726	* @param pHCPhys Where to re-initialize the physical address of the
727	* allocation as 0.
728	*/
729	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
730	{
731	AssertPtr(pMemObj);
732	AssertPtr(ppVirt);
733	AssertPtr(pHCPhys);
734	if (*pMemObj != NIL_RTR0MEMOBJ)
735	{
736	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
737	AssertRC(rc);
738	*pMemObj = NIL_RTR0MEMOBJ;
739	*ppVirt = 0;
740	*pHCPhys = 0;
741	}
742	}
743
744
745	/**
746	* Worker function to free VT-x related structures.
747	*
748	* @returns IPRT status code.
749	* @param pVM Pointer to the VM.
750	*/
751	static void hmR0VmxStructsFree(PVM pVM)
752	{
753	for (VMCPUID i = 0; i < pVM->cCpus; i++)
754	{
755	PVMCPU pVCpu = &pVM->aCpus[i];
756	AssertPtr(pVCpu);
757
758	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
759	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
761	#endif
762
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
765
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
768	}
769
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
772	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
773	#endif
774	}
775
776
777	/**
778	* Worker function to allocate VT-x related VM structures.
779	*
780	* @returns IPRT status code.
781	* @param pVM Pointer to the VM.
782	*/
783	static int hmR0VmxStructsAlloc(PVM pVM)
784	{
785	/*
786	* Initialize members up-front so we can cleanup properly on allocation failure.
787	*/
788	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
789	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
790	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
791	pVM->hm.s.vmx.HCPhys##a_Name = 0;
792
793	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
794	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
795	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
796	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
797
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
800	#endif
801	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
802
803	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
804	for (VMCPUID i = 0; i < pVM->cCpus; i++)
805	{
806	PVMCPU pVCpu = &pVM->aCpus[i];
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
813	#endif
814	}
815	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
816	#undef VMXLOCAL_INIT_VM_MEMOBJ
817
818	/*
819	* Allocate all the VT-x structures.
820	*/
821	int rc = VINF_SUCCESS;
822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
823	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
827	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
828	#endif
829
830	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
831	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
832	{
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
834	&pVM->hm.s.vmx.HCPhysApicAccess);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	/*
840	* Initialize per-VCPU VT-x structures.
841	*/
842	for (VMCPUID i = 0; i < pVM->cCpus; i++)
843	{
844	PVMCPU pVCpu = &pVM->aCpus[i];
845	AssertPtr(pVCpu);
846
847	/* Allocate the VM control structure (VMCS). */
848	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
854	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
855	{
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
857	&pVCpu->hm.s.vmx.HCPhysVirtApic);
858	if (RT_FAILURE(rc))
859	goto cleanup;
860	}
861
862	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
864	{
865	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
866	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867	if (RT_FAILURE(rc))
868	goto cleanup;
869	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
870	}
871
872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
873	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
874	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875	if (RT_FAILURE(rc))
876	goto cleanup;
877
878	/* Allocate the VM-exit MSR-load page for the host MSRs. */
879	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	#endif
883	}
884
885	return VINF_SUCCESS;
886
887	cleanup:
888	hmR0VmxStructsFree(pVM);
889	return rc;
890	}
891
892
893	/**
894	* Does global VT-x initialization (called during module initialization).
895	*
896	* @returns VBox status code.
897	*/
898	VMMR0DECL(int) VMXR0GlobalInit(void)
899	{
900	#ifdef HMVMX_USE_FUNCTION_TABLE
901	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
902	# ifdef VBOX_STRICT
903	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
904	Assert(g_apfnVMExitHandlers[i]);
905	# endif
906	#endif
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Does global VT-x termination (called during module termination).
913	*/
914	VMMR0DECL(void) VMXR0GlobalTerm()
915	{
916	/* Nothing to do currently. */
917	}
918
919
920	/**
921	* Sets up and activates VT-x on the current CPU.
922	*
923	* @returns VBox status code.
924	* @param pCpu Pointer to the global CPU info struct.
925	* @param pVM Pointer to the VM (can be NULL after a host resume
926	* operation).
927	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
928	* fEnabledByHost is true).
929	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
930	* @a fEnabledByHost is true).
931	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
932	* enable VT-x on the host.
933	*/
934	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
935	{
936	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
937	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
938
939	if (!fEnabledByHost)
940	{
941	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
942	if (RT_FAILURE(rc))
943	return rc;
944	}
945
946	/*
947	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
948	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
949	* each time while reusing a VPID after hitting the MaxASID limit once.
950	*/
951	if ( pVM
952	&& pVM->hm.s.fNestedPaging)
953	{
954	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
955	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
956	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
957	pCpu->fFlushAsidBeforeUse = false;
958	}
959	else
960	{
961	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
962	* without Nested Paging triggered this function) we still have the risk
963	* of potentially running with stale TLB-entries from other hypervisors
964	* when later we use a VM with NestedPaging. To fix this properly we will
965	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
966	* 'vmx_ept_vpid_caps' from it. Sigh. */
967	pCpu->fFlushAsidBeforeUse = true;
968	}
969
970	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
971	++pCpu->cTlbFlushes;
972
973	return VINF_SUCCESS;
974	}
975
976
977	/**
978	* Deactivates VT-x on the current CPU.
979	*
980	* @returns VBox status code.
981	* @param pCpu Pointer to the global CPU info struct.
982	* @param pvCpuPage Pointer to the VMXON region.
983	* @param HCPhysCpuPage Physical address of the VMXON region.
984	*
985	* @remarks This function should never be called when SUPR0EnableVTx() or
986	* similar was used to enable VT-x on the host.
987	*/
988	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
989	{
990	NOREF(pCpu);
991	NOREF(pvCpuPage);
992	NOREF(HCPhysCpuPage);
993
994	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
995	return hmR0VmxLeaveRootMode();
996	}
997
998
999	/**
1000	* Sets the permission bits for the specified MSR in the MSR bitmap.
1001	*
1002	* @param pVCpu Pointer to the VMCPU.
1003	* @param uMSR The MSR value.
1004	* @param enmRead Whether reading this MSR causes a VM-exit.
1005	* @param enmWrite Whether writing this MSR causes a VM-exit.
1006	*/
1007	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1008	{
1009	int32_t iBit;
1010	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1011
1012	/*
1013	* Layout:
1014	* 0x000 - 0x3ff - Low MSR read bits
1015	* 0x400 - 0x7ff - High MSR read bits
1016	* 0x800 - 0xbff - Low MSR write bits
1017	* 0xc00 - 0xfff - High MSR write bits
1018	*/
1019	if (uMsr <= 0x00001FFF)
1020	iBit = uMsr;
1021	else if ( uMsr >= 0xC0000000
1022	&& uMsr <= 0xC0001FFF)
1023	{
1024	iBit = (uMsr - 0xC0000000);
1025	pbMsrBitmap += 0x400;
1026	}
1027	else
1028	{
1029	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1030	return;
1031	}
1032
1033	Assert(iBit <= 0x1fff);
1034	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1035	ASMBitSet(pbMsrBitmap, iBit);
1036	else
1037	ASMBitClear(pbMsrBitmap, iBit);
1038
1039	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1040	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1041	else
1042	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1043	}
1044
1045
1046	/**
1047	* Flushes the TLB using EPT.
1048	*
1049	* @returns VBox status code.
1050	* @param pVM Pointer to the VM.
1051	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1052	* enmFlush).
1053	* @param enmFlush Type of flush.
1054	*/
1055	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1056	{
1057	AssertPtr(pVM);
1058	Assert(pVM->hm.s.fNestedPaging);
1059
1060	uint64_t descriptor[2];
1061	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1062	descriptor[0] = 0;
1063	else
1064	{
1065	Assert(pVCpu);
1066	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1067	}
1068	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1069
1070	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1071	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1072	rc));
1073	if ( RT_SUCCESS(rc)
1074	&& pVCpu)
1075	{
1076	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1077	}
1078	}
1079
1080
1081	/**
1082	* Flushes the TLB using VPID.
1083	*
1084	* @returns VBox status code.
1085	* @param pVM Pointer to the VM.
1086	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1087	* enmFlush).
1088	* @param enmFlush Type of flush.
1089	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1090	* on @a enmFlush).
1091	*/
1092	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1093	{
1094	AssertPtr(pVM);
1095	Assert(pVM->hm.s.vmx.fVpid);
1096
1097	uint64_t descriptor[2];
1098	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1099	{
1100	descriptor[0] = 0;
1101	descriptor[1] = 0;
1102	}
1103	else
1104	{
1105	AssertPtr(pVCpu);
1106	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1107	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1108	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1109	descriptor[1] = GCPtr;
1110	}
1111
1112	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1113	AssertMsg(rc == VINF_SUCCESS,
1114	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1115	if ( RT_SUCCESS(rc)
1116	&& pVCpu)
1117	{
1118	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1119	}
1120	}
1121
1122
1123	/**
1124	* Invalidates a guest page by guest virtual address. Only relevant for
1125	* EPT/VPID, otherwise there is nothing really to invalidate.
1126	*
1127	* @returns VBox status code.
1128	* @param pVM Pointer to the VM.
1129	* @param pVCpu Pointer to the VMCPU.
1130	* @param GCVirt Guest virtual address of the page to invalidate.
1131	*/
1132	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1133	{
1134	AssertPtr(pVM);
1135	AssertPtr(pVCpu);
1136	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1137
1138	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1139	if (!fFlushPending)
1140	{
1141	/*
1142	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1143	* See @bugref{6043} and @bugref{6177}.
1144	*
1145	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1146	* function maybe called in a loop with individual addresses.
1147	*/
1148	if (pVM->hm.s.vmx.fVpid)
1149	{
1150	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1151	{
1152	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1153	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1154	}
1155	else
1156	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1157	}
1158	else if (pVM->hm.s.fNestedPaging)
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	}
1161
1162	return VINF_SUCCESS;
1163	}
1164
1165
1166	/**
1167	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1168	* otherwise there is nothing really to invalidate.
1169	*
1170	* @returns VBox status code.
1171	* @param pVM Pointer to the VM.
1172	* @param pVCpu Pointer to the VMCPU.
1173	* @param GCPhys Guest physical address of the page to invalidate.
1174	*/
1175	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1176	{
1177	LogFlowFunc(("%RGp\n", GCPhys));
1178
1179	/*
1180	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1181	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1182	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1183	*/
1184	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1186	return VINF_SUCCESS;
1187	}
1188
1189
1190	/**
1191	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1192	* case where neither EPT nor VPID is supported by the CPU.
1193	*
1194	* @param pVM Pointer to the VM.
1195	* @param pVCpu Pointer to the VMCPU.
1196	*
1197	* @remarks Called with interrupts disabled.
1198	*/
1199	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1200	{
1201	NOREF(pVM);
1202	AssertPtr(pVCpu);
1203	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1204	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1205
1206	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1207	AssertPtr(pCpu);
1208
1209	pVCpu->hm.s.TlbShootdown.cPages = 0;
1210	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1211	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1212	pVCpu->hm.s.fForceTLBFlush = false;
1213	return;
1214	}
1215
1216
1217	/**
1218	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1219	*
1220	* @param pVM Pointer to the VM.
1221	* @param pVCpu Pointer to the VMCPU.
1222	* @remarks All references to "ASID" in this function pertains to "VPID" in
1223	* Intel's nomenclature. The reason is, to avoid confusion in compare
1224	* statements since the host-CPU copies are named "ASID".
1225	*
1226	* @remarks Called with interrupts disabled.
1227	*/
1228	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1229	{
1230	#ifdef VBOX_WITH_STATISTICS
1231	bool fTlbFlushed = false;
1232	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1233	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1234	if (!fTlbFlushed) \
1235	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1236	} while (0)
1237	#else
1238	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1239	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1240	#endif
1241
1242	AssertPtr(pVM);
1243	AssertPtr(pVCpu);
1244	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1245	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1246	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1247
1248	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1249	AssertPtr(pCpu);
1250
1251	/*
1252	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1253	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1254	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1255	*/
1256	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1257	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1258	{
1259	++pCpu->uCurrentAsid;
1260	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1261	{
1262	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1263	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1264	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1265	}
1266
1267	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1268	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1269	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1270
1271	/*
1272	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1273	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1274	*/
1275	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1276	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1277	HMVMX_SET_TAGGED_TLB_FLUSHED();
1278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1279	}
1280
1281	/* Check for explicit TLB shootdowns. */
1282	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1283	{
1284	/*
1285	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1286	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1287	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1288	* but not guest-physical mappings.
1289	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1290	*/
1291	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1292	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1293	HMVMX_SET_TAGGED_TLB_FLUSHED();
1294	}
1295
1296	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1297	* not be executed. See hmQueueInvlPage() where it is commented
1298	* out. Support individual entry flushing someday. */
1299	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1300	{
1301	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1302
1303	/*
1304	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1305	* as supported by the CPU.
1306	*/
1307	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1308	{
1309	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1310	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1311	}
1312	else
1313	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1314
1315	HMVMX_SET_TAGGED_TLB_FLUSHED();
1316	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1317	}
1318
1319	pVCpu->hm.s.TlbShootdown.cPages = 0;
1320	pVCpu->hm.s.fForceTLBFlush = false;
1321
1322	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1323
1324	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1325	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1326	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1327	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1328	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1329	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1330	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1331	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1332
1333	/* Update VMCS with the VPID. */
1334	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1335	AssertRC(rc);
1336
1337	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1338	}
1339
1340
1341	/**
1342	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1343	*
1344	* @returns VBox status code.
1345	* @param pVM Pointer to the VM.
1346	* @param pVCpu Pointer to the VMCPU.
1347	*
1348	* @remarks Called with interrupts disabled.
1349	*/
1350	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1351	{
1352	AssertPtr(pVM);
1353	AssertPtr(pVCpu);
1354	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1355	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1356
1357	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1358	AssertPtr(pCpu);
1359
1360	/*
1361	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1362	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1363	*/
1364	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1365	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1366	{
1367	pVCpu->hm.s.fForceTLBFlush = true;
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1369	}
1370
1371	/* Check for explicit TLB shootdown flushes. */
1372	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1373	{
1374	pVCpu->hm.s.fForceTLBFlush = true;
1375	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1376	}
1377
1378	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1379	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1380
1381	if (pVCpu->hm.s.fForceTLBFlush)
1382	{
1383	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1384	pVCpu->hm.s.fForceTLBFlush = false;
1385	}
1386	else
1387	{
1388	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1389	* not be executed. See hmQueueInvlPage() where it is commented
1390	* out. Support individual entry flushing someday. */
1391	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1392	{
1393	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1394	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1395	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1396	}
1397	else
1398	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1399	}
1400
1401	pVCpu->hm.s.TlbShootdown.cPages = 0;
1402	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1403	}
1404
1405
1406	/**
1407	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1408	*
1409	* @returns VBox status code.
1410	* @param pVM Pointer to the VM.
1411	* @param pVCpu Pointer to the VMCPU.
1412	*
1413	* @remarks Called with interrupts disabled.
1414	*/
1415	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1416	{
1417	AssertPtr(pVM);
1418	AssertPtr(pVCpu);
1419	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1420	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1421
1422	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1423
1424	/*
1425	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1426	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1427	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1428	*/
1429	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1430	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1431	{
1432	pVCpu->hm.s.fForceTLBFlush = true;
1433	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1434	}
1435
1436	/* Check for explicit TLB shootdown flushes. */
1437	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1438	{
1439	/*
1440	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1441	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1442	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1443	*/
1444	pVCpu->hm.s.fForceTLBFlush = true;
1445	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1446	}
1447
1448	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1449	if (pVCpu->hm.s.fForceTLBFlush)
1450	{
1451	++pCpu->uCurrentAsid;
1452	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1453	{
1454	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1455	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1456	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1457	}
1458
1459	pVCpu->hm.s.fForceTLBFlush = false;
1460	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1461	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1462	if (pCpu->fFlushAsidBeforeUse)
1463	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1464	}
1465	else
1466	{
1467	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1468	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1469	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1470	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1471
1472	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1473	* not be executed. See hmQueueInvlPage() where it is commented
1474	* out. Support individual entry flushing someday. */
1475	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1476	{
1477	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1478	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1479	{
1480	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1481	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1482	}
1483	else
1484	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1485	}
1486	else
1487	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1488	}
1489
1490	pVCpu->hm.s.TlbShootdown.cPages = 0;
1491	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1492
1493	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1494	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1495	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1496	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1497	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1498	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1499
1500	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1501	AssertRC(rc);
1502	}
1503
1504
1505	/**
1506	* Flushes the guest TLB entry based on CPU capabilities.
1507	*
1508	* @param pVCpu Pointer to the VMCPU.
1509	*/
1510	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1511	{
1512	PVM pVM = pVCpu->CTX_SUFF(pVM);
1513	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1514	{
1515	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1516	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1517	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1518	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1519	default:
1520	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1521	break;
1522	}
1523	}
1524
1525
1526	/**
1527	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1528	* TLB entries from the host TLB before VM-entry.
1529	*
1530	* @returns VBox status code.
1531	* @param pVM Pointer to the VM.
1532	*/
1533	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1534	{
1535	/*
1536	* Determine optimal flush type for Nested Paging.
1537	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1538	* guest execution (see hmR3InitFinalizeR0()).
1539	*/
1540	if (pVM->hm.s.fNestedPaging)
1541	{
1542	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1543	{
1544	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1545	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1546	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1547	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1548	else
1549	{
1550	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1551	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1552	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1553	}
1554
1555	/* Make sure the write-back cacheable memory type for EPT is supported. */
1556	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1557	{
1558	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1559	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1560	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1561	}
1562	}
1563	else
1564	{
1565	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1566	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1567	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1568	}
1569	}
1570
1571	/*
1572	* Determine optimal flush type for VPID.
1573	*/
1574	if (pVM->hm.s.vmx.fVpid)
1575	{
1576	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1577	{
1578	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1579	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1580	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1581	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1582	else
1583	{
1584	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1585	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1586	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1587	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1588	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1589	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1590	pVM->hm.s.vmx.fVpid = false;
1591	}
1592	}
1593	else
1594	{
1595	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1596	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1597	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1598	pVM->hm.s.vmx.fVpid = false;
1599	}
1600	}
1601
1602	/*
1603	* Setup the handler for flushing tagged-TLBs.
1604	*/
1605	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1606	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1607	else if (pVM->hm.s.fNestedPaging)
1608	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1609	else if (pVM->hm.s.vmx.fVpid)
1610	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1611	else
1612	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1613	return VINF_SUCCESS;
1614	}
1615
1616
1617	/**
1618	* Sets up pin-based VM-execution controls in the VMCS.
1619	*
1620	* @returns VBox status code.
1621	* @param pVM Pointer to the VM.
1622	* @param pVCpu Pointer to the VMCPU.
1623	*/
1624	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1625	{
1626	AssertPtr(pVM);
1627	AssertPtr(pVCpu);
1628
1629	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1630	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1631
1632	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1633	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1634	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1635
1636	/* Enable the VMX preemption timer. */
1637	if (pVM->hm.s.vmx.fUsePreemptTimer)
1638	{
1639	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1640	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1641	}
1642
1643	if ((val & zap) != val)
1644	{
1645	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1646	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1647	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1648	}
1649
1650	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1651	AssertRCReturn(rc, rc);
1652
1653	/* Update VCPU with the currently set pin-based VM-execution controls. */
1654	pVCpu->hm.s.vmx.u32PinCtls = val;
1655	return rc;
1656	}
1657
1658
1659	/**
1660	* Sets up processor-based VM-execution controls in the VMCS.
1661	*
1662	* @returns VBox status code.
1663	* @param pVM Pointer to the VM.
1664	* @param pVMCPU Pointer to the VMCPU.
1665	*/
1666	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1667	{
1668	AssertPtr(pVM);
1669	AssertPtr(pVCpu);
1670
1671	int rc = VERR_INTERNAL_ERROR_5;
1672	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1673	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1674
1675	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1676	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1677	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1678	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1679	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1681	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1682
1683	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1684	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1685	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1686	{
1687	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1689	}
1690
1691	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1692	if (!pVM->hm.s.fNestedPaging)
1693	{
1694	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1695	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1696	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1697	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1698	}
1699
1700	/* Use TPR shadowing if supported by the CPU. */
1701	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1702	{
1703	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1704	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1706	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1707	AssertRCReturn(rc, rc);
1708
1709	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1710	/* CR8 writes causes a VM-exit based on TPR threshold. */
1711	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1712	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1713	}
1714	else
1715	{
1716	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1717	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1718	}
1719
1720	/* Use MSR-bitmaps if supported by the CPU. */
1721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1722	{
1723	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1724
1725	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1726	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1727	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1728	AssertRCReturn(rc, rc);
1729
1730	/*
1731	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1732	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1733	*/
1734	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1735	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1736	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1737	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1738	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1739	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1740	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1743	}
1744
1745	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1746	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1747	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1748
1749	if ((val & zap) != val)
1750	{
1751	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1752	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1753	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1754	}
1755
1756	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1757	AssertRCReturn(rc, rc);
1758
1759	/* Update VCPU with the currently set processor-based VM-execution controls. */
1760	pVCpu->hm.s.vmx.u32ProcCtls = val;
1761
1762	/*
1763	* Secondary processor-based VM-execution controls.
1764	*/
1765	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1766	{
1767	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1768	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1769
1770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1771	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1772
1773	if (pVM->hm.s.fNestedPaging)
1774	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1775	else
1776	{
1777	/*
1778	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1779	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1780	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1781	*/
1782	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1783	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1784	}
1785
1786	if (pVM->hm.s.vmx.fVpid)
1787	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1788
1789	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1790	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1791
1792	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1793	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1794	* done dynamically. */
1795	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1796	{
1797	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1798	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1799	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1800	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1801	AssertRCReturn(rc, rc);
1802	}
1803
1804	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1805	{
1806	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1807	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1808	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809	}
1810
1811	if ((val & zap) != val)
1812	{
1813	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1814	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1815	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1816	}
1817
1818	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1819	AssertRCReturn(rc, rc);
1820
1821	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1822	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1823	}
1824
1825	return VINF_SUCCESS;
1826	}
1827
1828
1829	/**
1830	* Sets up miscellaneous (everything other than Pin & Processor-based
1831	* VM-execution) control fields in the VMCS.
1832	*
1833	* @returns VBox status code.
1834	* @param pVM Pointer to the VM.
1835	* @param pVCpu Pointer to the VMCPU.
1836	*/
1837	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1838	{
1839	AssertPtr(pVM);
1840	AssertPtr(pVCpu);
1841
1842	int rc = VERR_GENERAL_FAILURE;
1843
1844	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1845	#if 0
1846	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1847	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1848	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1849
1850	/*
1851	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1852	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1853	* We thus use the exception bitmap to control it rather than use both.
1854	*/
1855	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1856	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1857
1858	/** @todo Explore possibility of using IO-bitmaps. */
1859	/* All IO & IOIO instructions cause VM-exits. */
1860	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1861	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1862
1863	/* Initialize the MSR-bitmap area. */
1864	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1865	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1866	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1867	#endif
1868
1869	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1870	/* Setup MSR autoloading/storing. */
1871	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1872	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1873	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1874	AssertRCReturn(rc, rc);
1875	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1876	AssertRCReturn(rc, rc);
1877
1878	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1879	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1880	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1881	AssertRCReturn(rc, rc);
1882	#endif
1883
1884	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1885	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1886	AssertRCReturn(rc, rc);
1887
1888	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1889	#if 0
1890	/* Setup debug controls */
1891	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1892	AssertRCReturn(rc, rc);
1893	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1894	AssertRCReturn(rc, rc);
1895	#endif
1896
1897	return rc;
1898	}
1899
1900
1901	/**
1902	* Sets up the initial exception bitmap in the VMCS based on static conditions
1903	* (i.e. conditions that cannot ever change at runtime).
1904	*
1905	* @returns VBox status code.
1906	* @param pVM Pointer to the VM.
1907	* @param pVCpu Pointer to the VMCPU.
1908	*/
1909	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1910	{
1911	AssertPtr(pVM);
1912	AssertPtr(pVCpu);
1913
1914	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1915
1916	uint32_t u32XcptBitmap = 0;
1917
1918	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1919	if (!pVM->hm.s.fNestedPaging)
1920	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1921
1922	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1923	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1924	AssertRCReturn(rc, rc);
1925	return rc;
1926	}
1927
1928
1929	/**
1930	* Sets up the initial guest-state mask. The guest-state mask is consulted
1931	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1932	* for the nested virtualization case (as it would cause a VM-exit).
1933	*
1934	* @param pVCpu Pointer to the VMCPU.
1935	*/
1936	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1937	{
1938	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1939	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1940	return VINF_SUCCESS;
1941	}
1942
1943
1944	/**
1945	* Does per-VM VT-x initialization.
1946	*
1947	* @returns VBox status code.
1948	* @param pVM Pointer to the VM.
1949	*/
1950	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1951	{
1952	LogFlowFunc(("pVM=%p\n", pVM));
1953
1954	int rc = hmR0VmxStructsAlloc(pVM);
1955	if (RT_FAILURE(rc))
1956	{
1957	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1958	return rc;
1959	}
1960
1961	return VINF_SUCCESS;
1962	}
1963
1964
1965	/**
1966	* Does per-VM VT-x termination.
1967	*
1968	* @returns VBox status code.
1969	* @param pVM Pointer to the VM.
1970	*/
1971	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1972	{
1973	LogFlowFunc(("pVM=%p\n", pVM));
1974
1975	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1976	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1977	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1978	#endif
1979	hmR0VmxStructsFree(pVM);
1980	return VINF_SUCCESS;
1981	}
1982
1983
1984	/**
1985	* Sets up the VM for execution under VT-x.
1986	* This function is only called once per-VM during initalization.
1987	*
1988	* @returns VBox status code.
1989	* @param pVM Pointer to the VM.
1990	*/
1991	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1992	{
1993	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1994	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1995
1996	LogFlowFunc(("pVM=%p\n", pVM));
1997
1998	/*
1999	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2000	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2001	*/
2002	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2003	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2004	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2005	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2006	{
2007	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2008	return VERR_INTERNAL_ERROR;
2009	}
2010
2011	/* Initialize these always, see hmR3InitFinalizeR0().*/
2012	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2013	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2014
2015	/* Setup the tagged-TLB flush handlers. */
2016	int rc = hmR0VmxSetupTaggedTlb(pVM);
2017	if (RT_FAILURE(rc))
2018	{
2019	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2020	return rc;
2021	}
2022
2023	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2024	{
2025	PVMCPU pVCpu = &pVM->aCpus[i];
2026	AssertPtr(pVCpu);
2027	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2028
2029	/* Set revision dword at the beginning of the VMCS structure. */
2030	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2031
2032	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2033	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2034	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2035	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2036
2037	/* Load this VMCS as the current VMCS. */
2038	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2039	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2040	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2041
2042	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2043	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2044	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2045
2046	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2047	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2048	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2049
2050	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2051	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2052	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2053
2054	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2055	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2056	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2057
2058	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2059	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2060	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2061
2062	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2063	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2064	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2065	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2066	#endif
2067
2068	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2069	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2070	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2071	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2072
2073	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2074	}
2075
2076	return VINF_SUCCESS;
2077	}
2078
2079
2080	/**
2081	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2082	* the VMCS.
2083	*
2084	* @returns VBox status code.
2085	* @param pVM Pointer to the VM.
2086	* @param pVCpu Pointer to the VMCPU.
2087	*/
2088	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2089	{
2090	RTCCUINTREG uReg = ASMGetCR0();
2091	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2092	AssertRCReturn(rc, rc);
2093
2094	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2095	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2096	if (HMVMX_IS_64BIT_HOST_MODE())
2097	{
2098	uint64_t uRegCR3 = HMR0Get64bitCR3();
2099	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2100	}
2101	else
2102	#endif
2103	{
2104	uReg = ASMGetCR3();
2105	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2106	}
2107	AssertRCReturn(rc, rc);
2108
2109	uReg = ASMGetCR4();
2110	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2111	AssertRCReturn(rc, rc);
2112	return rc;
2113	}
2114
2115
2116	/**
2117	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2118	* the host-state area in the VMCS.
2119	*
2120	* @returns VBox status code.
2121	* @param pVM Pointer to the VM.
2122	* @param pVCpu Pointer to the VMCPU.
2123	*/
2124	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2125	{
2126	int rc = VERR_INTERNAL_ERROR_5;
2127	RTSEL uSelDS = 0;
2128	RTSEL uSelES = 0;
2129	RTSEL uSelFS = 0;
2130	RTSEL uSelGS = 0;
2131	RTSEL uSelTR = 0;
2132
2133	/*
2134	* Host DS, ES, FS and GS segment registers.
2135	*/
2136	#if HC_ARCH_BITS == 64
2137	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2138	uSelDS = ASMGetDS();
2139	uSelES = ASMGetES();
2140	uSelFS = ASMGetFS();
2141	uSelGS = ASMGetGS();
2142	#endif
2143
2144	/*
2145	* Host CS and SS segment registers.
2146	*/
2147	RTSEL uSelCS;
2148	RTSEL uSelSS;
2149	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2150	if (HMVMX_IS_64BIT_HOST_MODE())
2151	{
2152	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2153	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2154	}
2155	else
2156	{
2157	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2158	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2159	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2160	}
2161	#else
2162	uSelCS = ASMGetCS();
2163	uSelSS = ASMGetSS();
2164	#endif
2165
2166	/*
2167	* Host TR segment register.
2168	*/
2169	uSelTR = ASMGetTR();
2170
2171	#if HC_ARCH_BITS == 64
2172	/*
2173	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2174	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2175	*/
2176	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2177	{
2178	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2179	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2180	uSelDS = 0;
2181	}
2182	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2183	{
2184	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2185	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2186	uSelES = 0;
2187	}
2188	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2189	{
2190	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2191	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2192	uSelFS = 0;
2193	}
2194	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2195	{
2196	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2197	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2198	uSelGS = 0;
2199	}
2200	#endif
2201
2202	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2203	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2204	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2205	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2206	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2207	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2208	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2209	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2210	Assert(uSelCS);
2211	Assert(uSelTR);
2212
2213	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2214	#if 0
2215	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2216	Assert(uSelSS != 0);
2217	#endif
2218
2219	/* Write these host selector fields into the host-state area in the VMCS. */
2220	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2221	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2222	#if HC_ARCH_BITS == 64
2223	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2224	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2225	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2226	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2227	#endif
2228	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2229
2230	/*
2231	* Host GDTR and IDTR.
2232	*/
2233	RTGDTR Gdtr;
2234	RT_ZERO(Gdtr);
2235	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2236	if (HMVMX_IS_64BIT_HOST_MODE())
2237	{
2238	X86XDTR64 Gdtr64;
2239	X86XDTR64 Idtr64;
2240	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2241	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2242	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2243
2244	Gdtr.cbGdt = Gdtr64.cb;
2245	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2246	}
2247	else
2248	#endif
2249	{
2250	RTIDTR Idtr;
2251	ASMGetGDTR(&Gdtr);
2252	ASMGetIDTR(&Idtr);
2253	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2254	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2255
2256	#if HC_ARCH_BITS == 64
2257	/*
2258	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2259	* maximum limit (0xffff) on every VM-exit.
2260	*/
2261	if (Gdtr.cbGdt != 0xffff)
2262	{
2263	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2264	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2265	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2266	}
2267
2268	/*
2269	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2270	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2271	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2272	*/
2273	if (Idtr.cbIdt < 0x0fff)
2274	{
2275	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2276	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2277	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2278	}
2279	#endif
2280	}
2281
2282	/*
2283	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2284	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2285	*/
2286	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2287	{
2288	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2289	return VERR_VMX_INVALID_HOST_STATE;
2290	}
2291
2292	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2293	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2294	if (HMVMX_IS_64BIT_HOST_MODE())
2295	{
2296	/* We need the 64-bit TR base for hybrid darwin. */
2297	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2298	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2299	}
2300	else
2301	#endif
2302	{
2303	uintptr_t uTRBase;
2304	#if HC_ARCH_BITS == 64
2305	uTRBase = X86DESC64_BASE(pDesc);
2306	#else
2307	uTRBase = X86DESC_BASE(pDesc);
2308	#endif
2309	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2310	}
2311	AssertRCReturn(rc, rc);
2312
2313	/*
2314	* Host FS base and GS base.
2315	*/
2316	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2317	if (HMVMX_IS_64BIT_HOST_MODE())
2318	{
2319	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2320	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2321	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2322	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2323
2324	# if HC_ARCH_BITS == 64
2325	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2326	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2327	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2328	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2329	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2330	# endif
2331	}
2332	#endif
2333	return rc;
2334	}
2335
2336
2337	/**
2338	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2339	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2340	* the host after every successful VM exit.
2341	*
2342	* @returns VBox status code.
2343	* @param pVM Pointer to the VM.
2344	* @param pVCpu Pointer to the VMCPU.
2345	*/
2346	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2347	{
2348	AssertPtr(pVCpu);
2349	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2350
2351	int rc = VINF_SUCCESS;
2352	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2353	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2354	uint32_t cHostMsrs = 0;
2355	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2356
2357	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2358	{
2359	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2360
2361	# if HC_ARCH_BITS == 64
2362	/* Paranoia. 64-bit code requires these bits to be set always. */
2363	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2364
2365	/*
2366	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2367	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2368	* some reason (e.g. allow transparent reads) we would activate the code below.
2369	*/
2370	# if 0
2371	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2372	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2373	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2374	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2375	if (CPUMIsGuestInLongMode(pVCpu))
2376	{
2377	uint64_t u64GuestEfer;
2378	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2379	AssertRC(rc);
2380
2381	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2382	{
2383	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2384	pHostMsr->u32Reserved = 0;
2385	pHostMsr->u64Value = u64HostEfer;
2386	pHostMsr++; cHostMsrs++;
2387	}
2388	}
2389	# endif
2390	# else /* HC_ARCH_BITS != 64 */
2391	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2392	pHostMsr->u32Reserved = 0;
2393	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2394	if (CPUMIsGuestInLongMode(pVCpu))
2395	{
2396	/* Must match the EFER value in our 64 bits switcher. */
2397	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2398	}
2399	else
2400	# endif
2401	pHostMsr->u64Value = u64HostEfer;
2402	pHostMsr++; cHostMsrs++;
2403	# endif /* HC_ARCH_BITS == 64 */
2404	}
2405
2406	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2407	if (HMVMX_IS_64BIT_HOST_MODE())
2408	{
2409	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2410	pHostMsr->u32Reserved = 0;
2411	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2412	pHostMsr++; cHostMsrs++;
2413	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2414	pHostMsr->u32Reserved = 0;
2415	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2416	pHostMsr++; cHostMsrs++;
2417	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2418	pHostMsr->u32Reserved = 0;
2419	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2420	pHostMsr++; cHostMsrs++;
2421	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2422	pHostMsr->u32Reserved = 0;
2423	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2424	pHostMsr++; cHostMsrs++;
2425	}
2426	# endif
2427
2428	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2429	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2430	{
2431	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2432	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2433	}
2434
2435	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2436	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2437
2438	/*
2439	* Host Sysenter MSRs.
2440	*/
2441	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2442	AssertRCReturn(rc, rc);
2443	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2444	if (HMVMX_IS_64BIT_HOST_MODE())
2445	{
2446	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2447	AssertRCReturn(rc, rc);
2448	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2449	}
2450	else
2451	{
2452	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2453	AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2455	}
2456	#elif HC_ARCH_BITS == 32
2457	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2458	AssertRCReturn(rc, rc);
2459	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2460	#else
2461	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2462	AssertRCReturn(rc, rc);
2463	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2464	#endif
2465	AssertRCReturn(rc, rc);
2466
2467	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2468	* hmR0VmxSetupExitCtls() !! */
2469	return rc;
2470	}
2471
2472
2473	/**
2474	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2475	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2476	* controls".
2477	*
2478	* @returns VBox status code.
2479	* @param pVCpu Pointer to the VMCPU.
2480	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2481	* out-of-sync. Make sure to update the required fields
2482	* before using them.
2483	*
2484	* @remarks No-long-jump zone!!!
2485	*/
2486	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2487	{
2488	int rc = VINF_SUCCESS;
2489	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2490	{
2491	PVM pVM = pVCpu->CTX_SUFF(pVM);
2492	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2493	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2494
2495	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2496	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2497
2498	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2499	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2500	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2501	else
2502	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2503
2504	/*
2505	* The following should not be set (since we're not in SMM mode):
2506	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2507	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2508	*/
2509
2510	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2511	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2512	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2513
2514	if ((val & zap) != val)
2515	{
2516	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2517	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2518	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2519	}
2520
2521	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2522	AssertRCReturn(rc, rc);
2523
2524	/* Update VCPU with the currently set VM-exit controls. */
2525	pVCpu->hm.s.vmx.u32EntryCtls = val;
2526	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2527	}
2528	return rc;
2529	}
2530
2531
2532	/**
2533	* Sets up the VM-exit controls in the VMCS.
2534	*
2535	* @returns VBox status code.
2536	* @param pVM Pointer to the VM.
2537	* @param pVCpu Pointer to the VMCPU.
2538	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2539	* out-of-sync. Make sure to update the required fields
2540	* before using them.
2541	*
2542	* @remarks requires EFER.
2543	*/
2544	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2545	{
2546	int rc = VINF_SUCCESS;
2547	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2548	{
2549	PVM pVM = pVCpu->CTX_SUFF(pVM);
2550	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2551	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2552
2553	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2554	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2555
2556	/*
2557	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2558	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2559	*/
2560	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2561	if (HMVMX_IS_64BIT_HOST_MODE())
2562	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2563	else
2564	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2565	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2566	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2567	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2568	else
2569	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2570	#endif
2571
2572	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2573	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2574
2575	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2576	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2577	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2578	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2579	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2580
2581	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2582	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2583
2584	if ((val & zap) != val)
2585	{
2586	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2587	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2588	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2589	}
2590
2591	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2592	AssertRCReturn(rc, rc);
2593
2594	/* Update VCPU with the currently set VM-exit controls. */
2595	pVCpu->hm.s.vmx.u32ExitCtls = val;
2596	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2597	}
2598	return rc;
2599	}
2600
2601
2602	/**
2603	* Loads the guest APIC and related state.
2604	*
2605	* @returns VBox status code.
2606	* @param pVM Pointer to the VM.
2607	* @param pVCpu Pointer to the VMCPU.
2608	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2609	* out-of-sync. Make sure to update the required fields
2610	* before using them.
2611	*/
2612	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2613	{
2614	int rc = VINF_SUCCESS;
2615	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2616	{
2617	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2618	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2619	{
2620	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2621
2622	bool fPendingIntr = false;
2623	uint8_t u8Tpr = 0;
2624	uint8_t u8PendingIntr = 0;
2625	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2626	AssertRCReturn(rc, rc);
2627
2628	/*
2629	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2630	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2631	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2632	* the interrupt when we VM-exit for other reasons.
2633	*/
2634	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2635	uint32_t u32TprThreshold = 0;
2636	if (fPendingIntr)
2637	{
2638	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2639	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2640	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2641	if (u8PendingPriority <= u8TprPriority)
2642	u32TprThreshold = u8PendingPriority;
2643	else
2644	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2645	}
2646	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2647
2648	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2649	AssertRCReturn(rc, rc);
2650
2651	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2652	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2653	{
2654	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2655	pMixedCtx->msrLSTAR = u8Tpr;
2656	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2657	{
2658	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2659	if (fPendingIntr)
2660	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2661	else
2662	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2663	}
2664	}
2665	}
2666
2667	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2668	}
2669	return rc;
2670	}
2671
2672
2673	/**
2674	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2675	*
2676	* @returns Guest's interruptibility-state.
2677	* @param pVCpu Pointer to the VMCPU.
2678	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2679	* out-of-sync. Make sure to update the required fields
2680	* before using them.
2681	*
2682	* @remarks No-long-jump zone!!!
2683	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2684	*/
2685	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2686	{
2687	/*
2688	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2689	* inhibit interrupts or clear any existing interrupt-inhibition.
2690	*/
2691	uint32_t uIntrState = 0;
2692	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2693	{
2694	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2695	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2696	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2697	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2698	{
2699	/*
2700	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2701	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2702	*/
2703	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2704	}
2705	else if (pMixedCtx->eflags.Bits.u1IF)
2706	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2707	else
2708	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2709	}
2710	return uIntrState;
2711	}
2712
2713
2714	/**
2715	* Loads the guest's interruptibility-state into the guest-state area in the
2716	* VMCS.
2717	*
2718	* @returns VBox status code.
2719	* @param pVCpu Pointer to the VMCPU.
2720	* @param uIntrState The interruptibility-state to set.
2721	*/
2722	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2723	{
2724	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2725	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2726	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2727	AssertRCReturn(rc, rc);
2728	return rc;
2729	}
2730
2731
2732	/**
2733	* Loads the guest's RIP into the guest-state area in the VMCS.
2734	*
2735	* @returns VBox status code.
2736	* @param pVCpu Pointer to the VMCPU.
2737	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2738	* out-of-sync. Make sure to update the required fields
2739	* before using them.
2740	*
2741	* @remarks No-long-jump zone!!!
2742	*/
2743	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2744	{
2745	int rc = VINF_SUCCESS;
2746	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2747	{
2748	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2749	AssertRCReturn(rc, rc);
2750	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2751	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2752	}
2753	return rc;
2754	}
2755
2756
2757	/**
2758	* Loads the guest's RSP into the guest-state area in the VMCS.
2759	*
2760	* @returns VBox status code.
2761	* @param pVCpu Pointer to the VMCPU.
2762	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2763	* out-of-sync. Make sure to update the required fields
2764	* before using them.
2765	*
2766	* @remarks No-long-jump zone!!!
2767	*/
2768	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2769	{
2770	int rc = VINF_SUCCESS;
2771	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2772	{
2773	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2774	AssertRCReturn(rc, rc);
2775	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2776	}
2777	return rc;
2778	}
2779
2780
2781	/**
2782	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2783	*
2784	* @returns VBox status code.
2785	* @param pVCpu Pointer to the VMCPU.
2786	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2787	* out-of-sync. Make sure to update the required fields
2788	* before using them.
2789	*
2790	* @remarks No-long-jump zone!!!
2791	*/
2792	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2793	{
2794	int rc = VINF_SUCCESS;
2795	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2796	{
2797	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2798	Let us assert it as such and use 32-bit VMWRITE. */
2799	Assert(!(pMixedCtx->rflags.u64 >> 32));
2800	X86EFLAGS uEFlags = pMixedCtx->eflags;
2801	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2802	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2803
2804	/*
2805	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2806	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2807	*/
2808	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2809	{
2810	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2811	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2812	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2813	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2814	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2815	}
2816
2817	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2818	AssertRCReturn(rc, rc);
2819
2820	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2821	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2822	}
2823	return rc;
2824	}
2825
2826
2827	/**
2828	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2829	*
2830	* @returns VBox status code.
2831	* @param pVCpu Pointer to the VMCPU.
2832	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2833	* out-of-sync. Make sure to update the required fields
2834	* before using them.
2835	*
2836	* @remarks No-long-jump zone!!!
2837	*/
2838	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2839	{
2840	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2841	AssertRCReturn(rc, rc);
2842	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2843	AssertRCReturn(rc, rc);
2844	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2845	AssertRCReturn(rc, rc);
2846	return rc;
2847	}
2848
2849
2850	/**
2851	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2852	* in the VMCS.
2853	*
2854	* @returns VBox status code.
2855	* @param pVM Pointer to the VM.
2856	* @param pVCpu Pointer to the VMCPU.
2857	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2858	* out-of-sync. Make sure to update the required fields
2859	* before using them.
2860	*
2861	* @remarks No-long-jump zone!!!
2862	*/
2863	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2864	{
2865	int rc = VINF_SUCCESS;
2866	PVM pVM = pVCpu->CTX_SUFF(pVM);
2867
2868	/*
2869	* Guest CR0.
2870	* Guest FPU.
2871	*/
2872	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2873	{
2874	Assert(!(pCtx->cr0 >> 32));
2875	uint32_t u32GuestCR0 = pCtx->cr0;
2876
2877	/* The guest's view (read access) of its CR0 is unblemished. */
2878	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2879	AssertRCReturn(rc, rc);
2880	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2881
2882	/* Setup VT-x's view of the guest CR0. */
2883	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2884	if (pVM->hm.s.fNestedPaging)
2885	{
2886	if (CPUMIsGuestPagingEnabledEx(pCtx))
2887	{
2888	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2889	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2890	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2891	}
2892	else
2893	{
2894	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2895	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2896	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2897	}
2898
2899	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2900	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2901	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2902
2903	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2904	AssertRCReturn(rc, rc);
2905	}
2906	else
2907	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2908
2909	/*
2910	* Guest FPU bits.
2911	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2912	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2913	*/
2914	u32GuestCR0 \|= X86_CR0_NE;
2915	bool fInterceptNM = false;
2916	if (CPUMIsGuestFPUStateActive(pVCpu))
2917	{
2918	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2919	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2920	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2921	}
2922	else
2923	{
2924	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2925	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2926	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2927	}
2928
2929	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2930	bool fInterceptMF = false;
2931	if (!(pCtx->cr0 & X86_CR0_NE))
2932	fInterceptMF = true;
2933
2934	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2935	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2936	{
2937	Assert(PDMVmmDevHeapIsEnabled(pVM));
2938	Assert(pVM->hm.s.vmx.pRealModeTSS);
2939	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2940	fInterceptNM = true;
2941	fInterceptMF = true;
2942	}
2943	else
2944	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2945
2946	if (fInterceptNM)
2947	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2948	else
2949	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2950
2951	if (fInterceptMF)
2952	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2953	else
2954	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2955
2956	/* Additional intercepts for debugging, define these yourself explicitly. */
2957	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2958	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2959	\| RT_BIT(X86_XCPT_DB)
2960	\| RT_BIT(X86_XCPT_DE)
2961	\| RT_BIT(X86_XCPT_NM)
2962	\| RT_BIT(X86_XCPT_UD)
2963	\| RT_BIT(X86_XCPT_NP)
2964	\| RT_BIT(X86_XCPT_SS)
2965	\| RT_BIT(X86_XCPT_GP)
2966	\| RT_BIT(X86_XCPT_PF)
2967	\| RT_BIT(X86_XCPT_MF);
2968	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2969	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2970	#endif
2971
2972	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2973
2974	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2975	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2976	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2977	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2978	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2979	else
2980	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2981
2982	u32GuestCR0 \|= uSetCR0;
2983	u32GuestCR0 &= uZapCR0;
2984	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2985
2986	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2987	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2988	AssertRCReturn(rc, rc);
2989	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2990	AssertRCReturn(rc, rc);
2991	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2992
2993	/*
2994	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2995	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2996	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2997	*/
2998	uint32_t u32CR0Mask = 0;
2999	u32CR0Mask = X86_CR0_PE
3000	\| X86_CR0_NE
3001	\| X86_CR0_WP
3002	\| X86_CR0_PG
3003	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3004	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3005	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3006	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3007	u32CR0Mask &= ~X86_CR0_PE;
3008	if (pVM->hm.s.fNestedPaging)
3009	u32CR0Mask &= ~X86_CR0_WP;
3010
3011	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3012	if (fInterceptNM)
3013	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3014	else
3015	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3016
3017	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3018	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3019	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3020	AssertRCReturn(rc, rc);
3021
3022	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3023	}
3024
3025	/*
3026	* Guest CR2.
3027	* It's always loaded in the assembler code. Nothing to do here.
3028	*/
3029
3030	/*
3031	* Guest CR3.
3032	*/
3033	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3034	{
3035	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3036	if (pVM->hm.s.fNestedPaging)
3037	{
3038	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3039
3040	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3041	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3042	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3043	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3044
3045	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3046	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3047	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3048
3049	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3050	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3051	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3052	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3053
3054	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3055	AssertRCReturn(rc, rc);
3056	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3057
3058	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3059	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3060	{
3061	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3062	if (CPUMIsGuestInPAEModeEx(pCtx))
3063	{
3064	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3065	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3066	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3067	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3068	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3069	}
3070
3071	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3072	have Unrestricted Execution to handle the guest when it's not using paging. */
3073	GCPhysGuestCR3 = pCtx->cr3;
3074	}
3075	else
3076	{
3077	/*
3078	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3079	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3080	* EPT takes care of translating it to host-physical addresses.
3081	*/
3082	RTGCPHYS GCPhys;
3083	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3084	Assert(PDMVmmDevHeapIsEnabled(pVM));
3085
3086	/* We obtain it here every time as the guest could have relocated this PCI region. */
3087	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3088	AssertRCReturn(rc, rc);
3089
3090	GCPhysGuestCR3 = GCPhys;
3091	}
3092
3093	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3094	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3095	}
3096	else
3097	{
3098	/* Non-nested paging case, just use the hypervisor's CR3. */
3099	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3100
3101	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3102	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3103	}
3104	AssertRCReturn(rc, rc);
3105
3106	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3107	}
3108
3109	/*
3110	* Guest CR4.
3111	*/
3112	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3113	{
3114	Assert(!(pCtx->cr4 >> 32));
3115	uint32_t u32GuestCR4 = pCtx->cr4;
3116
3117	/* The guest's view of its CR4 is unblemished. */
3118	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3119	AssertRCReturn(rc, rc);
3120	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3121
3122	/* Setup VT-x's view of the guest CR4. */
3123	/*
3124	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3125	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3126	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3127	*/
3128	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3129	{
3130	Assert(pVM->hm.s.vmx.pRealModeTSS);
3131	Assert(PDMVmmDevHeapIsEnabled(pVM));
3132	u32GuestCR4 &= ~X86_CR4_VME;
3133	}
3134
3135	if (pVM->hm.s.fNestedPaging)
3136	{
3137	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3138	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3139	{
3140	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3141	u32GuestCR4 \|= X86_CR4_PSE;
3142	/* Our identity mapping is a 32 bits page directory. */
3143	u32GuestCR4 &= ~X86_CR4_PAE;
3144	}
3145	/* else use guest CR4.*/
3146	}
3147	else
3148	{
3149	/*
3150	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3151	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3152	*/
3153	switch (pVCpu->hm.s.enmShadowMode)
3154	{
3155	case PGMMODE_REAL: /* Real-mode. */
3156	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3157	case PGMMODE_32_BIT: /* 32-bit paging. */
3158	{
3159	u32GuestCR4 &= ~X86_CR4_PAE;
3160	break;
3161	}
3162
3163	case PGMMODE_PAE: /* PAE paging. */
3164	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3165	{
3166	u32GuestCR4 \|= X86_CR4_PAE;
3167	break;
3168	}
3169
3170	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3171	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3172	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3173	break;
3174	#endif
3175	default:
3176	AssertFailed();
3177	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3178	}
3179	}
3180
3181	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3182	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3183	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3184	u32GuestCR4 \|= uSetCR4;
3185	u32GuestCR4 &= uZapCR4;
3186
3187	/* Write VT-x's view of the guest CR4 into the VMCS. */
3188	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3189	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3190	AssertRCReturn(rc, rc);
3191
3192	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3193	uint32_t u32CR4Mask = 0;
3194	u32CR4Mask = X86_CR4_VME
3195	\| X86_CR4_PAE
3196	\| X86_CR4_PGE
3197	\| X86_CR4_PSE
3198	\| X86_CR4_VMXE;
3199	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3200	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3201	AssertRCReturn(rc, rc);
3202
3203	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3204	}
3205	return rc;
3206	}
3207
3208
3209	/**
3210	* Loads the guest debug registers into the guest-state area in the VMCS.
3211	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3212	*
3213	* @returns VBox status code.
3214	* @param pVCpu Pointer to the VMCPU.
3215	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3216	* out-of-sync. Make sure to update the required fields
3217	* before using them.
3218	*
3219	* @remarks No-long-jump zone!!!
3220	*/
3221	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3222	{
3223	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3224	return VINF_SUCCESS;
3225
3226	#ifdef VBOX_STRICT
3227	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3228	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3229	{
3230	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3231	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3232	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3233	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3234	}
3235	#endif
3236
3237	int rc = VERR_INTERNAL_ERROR_5;
3238	PVM pVM = pVCpu->CTX_SUFF(pVM);
3239	bool fInterceptDB = false;
3240	bool fInterceptMovDRx = false;
3241	if (DBGFIsStepping(pVCpu))
3242	{
3243	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3244	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3245	{
3246	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3247	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3248	AssertRCReturn(rc, rc);
3249	Assert(fInterceptDB == false);
3250	}
3251	else
3252	{
3253	fInterceptDB = true;
3254	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3255	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3256	}
3257	}
3258
3259	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3260	{
3261	if (!CPUMIsHyperDebugStateActive(pVCpu))
3262	{
3263	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3264	AssertRC(rc);
3265	}
3266	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3267	fInterceptMovDRx = true;
3268	}
3269	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3270	{
3271	if (!CPUMIsGuestDebugStateActive(pVCpu))
3272	{
3273	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3274	AssertRC(rc);
3275	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3276	}
3277	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3278	Assert(fInterceptMovDRx == false);
3279	}
3280	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3281	{
3282	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3283	fInterceptMovDRx = true;
3284	}
3285
3286	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3287	if (fInterceptDB)
3288	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3289	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3290	{
3291	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3292	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3293	#endif
3294	}
3295
3296	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3297	if (fInterceptMovDRx)
3298	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3299	else
3300	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3301
3302	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3303	AssertRCReturn(rc, rc);
3304	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3305	AssertRCReturn(rc, rc);
3306
3307	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3308	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3309	AssertRCReturn(rc, rc);
3310
3311	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3312	return rc;
3313	}
3314
3315
3316	#ifdef VBOX_STRICT
3317	/**
3318	* Strict function to validate segment registers.
3319	*
3320	* @remarks Requires CR0.
3321	*/
3322	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3323	{
3324	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3325	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3326	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3327
3328	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3329	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3330	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3331	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3332	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3333	{
3334	/* Protected mode checks */
3335	/* CS */
3336	Assert(pCtx->cs.Attr.n.u1Present);
3337	Assert(!(pCtx->cs.Attr.u & 0xf00));
3338	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3339	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3340	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3341	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3342	\|\| (pCtx->cs.Attr.n.u1Granularity));
3343	/* CS cannot be loaded with NULL in protected mode. */
3344	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3345	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3346	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3347	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3348	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3349	else
3350	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3351	/* SS */
3352	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3353	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3354	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3355	if ( !(pCtx->cr0 & X86_CR0_PE)
3356	\|\| pCtx->cs.Attr.n.u4Type == 3)
3357	{
3358	Assert(!pCtx->ss.Attr.n.u2Dpl);
3359	}
3360	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3361	{
3362	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3363	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3364	Assert(pCtx->ss.Attr.n.u1Present);
3365	Assert(!(pCtx->ss.Attr.u & 0xf00));
3366	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3367	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3368	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3369	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3370	\|\| (pCtx->ss.Attr.n.u1Granularity));
3371	}
3372	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3373	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3374	{
3375	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3376	Assert(pCtx->ds.Attr.n.u1Present);
3377	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3378	Assert(!(pCtx->ds.Attr.u & 0xf00));
3379	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3380	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3381	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3382	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3383	\|\| (pCtx->ds.Attr.n.u1Granularity));
3384	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3385	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3386	}
3387	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3388	{
3389	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3390	Assert(pCtx->es.Attr.n.u1Present);
3391	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3392	Assert(!(pCtx->es.Attr.u & 0xf00));
3393	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3394	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3395	\|\| !(pCtx->es.Attr.n.u1Granularity));
3396	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3397	\|\| (pCtx->es.Attr.n.u1Granularity));
3398	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3399	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3400	}
3401	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3402	{
3403	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3404	Assert(pCtx->fs.Attr.n.u1Present);
3405	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3406	Assert(!(pCtx->fs.Attr.u & 0xf00));
3407	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3408	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3409	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3410	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3411	\|\| (pCtx->fs.Attr.n.u1Granularity));
3412	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3413	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3414	}
3415	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3416	{
3417	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3418	Assert(pCtx->gs.Attr.n.u1Present);
3419	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3420	Assert(!(pCtx->gs.Attr.u & 0xf00));
3421	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3422	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3423	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3424	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3425	\|\| (pCtx->gs.Attr.n.u1Granularity));
3426	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3427	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3428	}
3429	/* 64-bit capable CPUs. */
3430	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3431	Assert(!(pCtx->cs.u64Base >> 32));
3432	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3433	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3434	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3435	# endif
3436	}
3437	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3438	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3439	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3440	{
3441	/* Real and v86 mode checks. */
3442	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3443	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3444	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3445	{
3446	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3447	}
3448	else
3449	{
3450	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3451	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3452	}
3453
3454	/* CS */
3455	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3456	Assert(pCtx->cs.u32Limit == 0xffff);
3457	Assert(u32CSAttr == 0xf3);
3458	/* SS */
3459	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3460	Assert(pCtx->ss.u32Limit == 0xffff);
3461	Assert(u32SSAttr == 0xf3);
3462	/* DS */
3463	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3464	Assert(pCtx->ds.u32Limit == 0xffff);
3465	Assert(u32DSAttr == 0xf3);
3466	/* ES */
3467	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3468	Assert(pCtx->es.u32Limit == 0xffff);
3469	Assert(u32ESAttr == 0xf3);
3470	/* FS */
3471	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3472	Assert(pCtx->fs.u32Limit == 0xffff);
3473	Assert(u32FSAttr == 0xf3);
3474	/* GS */
3475	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3476	Assert(pCtx->gs.u32Limit == 0xffff);
3477	Assert(u32GSAttr == 0xf3);
3478	/* 64-bit capable CPUs. */
3479	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3480	Assert(!(pCtx->cs.u64Base >> 32));
3481	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3482	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3483	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3484	# endif
3485	}
3486	}
3487	#endif /* VBOX_STRICT */
3488
3489
3490	/**
3491	* Writes a guest segment register into the guest-state area in the VMCS.
3492	*
3493	* @returns VBox status code.
3494	* @param pVCpu Pointer to the VMCPU.
3495	* @param idxSel Index of the selector in the VMCS.
3496	* @param idxLimit Index of the segment limit in the VMCS.
3497	* @param idxBase Index of the segment base in the VMCS.
3498	* @param idxAccess Index of the access rights of the segment in the VMCS.
3499	* @param pSelReg Pointer to the segment selector.
3500	* @param pCtx Pointer to the guest-CPU context.
3501	*
3502	* @remarks No-long-jump zone!!!
3503	*/
3504	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3505	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3506	{
3507	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3508	AssertRCReturn(rc, rc);
3509	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3510	AssertRCReturn(rc, rc);
3511	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3512	AssertRCReturn(rc, rc);
3513
3514	uint32_t u32Access = pSelReg->Attr.u;
3515	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3516	{
3517	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3518	u32Access = 0xf3;
3519	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3520	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3521	}
3522	else
3523	{
3524	/*
3525	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3526	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3527	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3528	* loaded in protected-mode have their attribute as 0.
3529	*/
3530	if (!u32Access)
3531	u32Access = HMVMX_SEL_UNUSABLE;
3532	}
3533
3534	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3535	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3536	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3537
3538	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3539	AssertRCReturn(rc, rc);
3540	return rc;
3541	}
3542
3543
3544	/**
3545	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3546	* into the guest-state area in the VMCS.
3547	*
3548	* @returns VBox status code.
3549	* @param pVM Pointer to the VM.
3550	* @param pVCPU Pointer to the VMCPU.
3551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3552	* out-of-sync. Make sure to update the required fields
3553	* before using them.
3554	*
3555	* @remarks Requires CR0 (strict builds validation).
3556	* @remarks No-long-jump zone!!!
3557	*/
3558	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3559	{
3560	int rc = VERR_INTERNAL_ERROR_5;
3561	PVM pVM = pVCpu->CTX_SUFF(pVM);
3562
3563	/*
3564	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3565	*/
3566	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3567	{
3568	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3569	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3570	{
3571	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3572	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3573	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3574	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3575	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3576	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3577	}
3578
3579	#ifdef VBOX_WITH_REM
3580	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3581	{
3582	Assert(pVM->hm.s.vmx.pRealModeTSS);
3583	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3584	if ( pVCpu->hm.s.vmx.fWasInRealMode
3585	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3586	{
3587	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3588	in real-mode (e.g. OpenBSD 4.0) */
3589	REMFlushTBs(pVM);
3590	Log4(("Load: Switch to protected mode detected!\n"));
3591	pVCpu->hm.s.vmx.fWasInRealMode = false;
3592	}
3593	}
3594	#endif
3595	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3596	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3597	AssertRCReturn(rc, rc);
3598	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3599	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3600	AssertRCReturn(rc, rc);
3601	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3602	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3603	AssertRCReturn(rc, rc);
3604	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3605	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3606	AssertRCReturn(rc, rc);
3607	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3608	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3609	AssertRCReturn(rc, rc);
3610	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3611	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3612	AssertRCReturn(rc, rc);
3613
3614	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3615	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3616	#ifdef VBOX_STRICT
3617	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3618	#endif
3619	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3620	}
3621
3622	/*
3623	* Guest TR.
3624	*/
3625	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3626	{
3627	/*
3628	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3629	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3630	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3631	*/
3632	uint16_t u16Sel = 0;
3633	uint32_t u32Limit = 0;
3634	uint64_t u64Base = 0;
3635	uint32_t u32AccessRights = 0;
3636
3637	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3638	{
3639	u16Sel = pMixedCtx->tr.Sel;
3640	u32Limit = pMixedCtx->tr.u32Limit;
3641	u64Base = pMixedCtx->tr.u64Base;
3642	u32AccessRights = pMixedCtx->tr.Attr.u;
3643	}
3644	else
3645	{
3646	Assert(pVM->hm.s.vmx.pRealModeTSS);
3647	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3648
3649	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3650	RTGCPHYS GCPhys;
3651	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3652	AssertRCReturn(rc, rc);
3653
3654	X86DESCATTR DescAttr;
3655	DescAttr.u = 0;
3656	DescAttr.n.u1Present = 1;
3657	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3658
3659	u16Sel = 0;
3660	u32Limit = HM_VTX_TSS_SIZE;
3661	u64Base = GCPhys; /* in real-mode phys = virt. */
3662	u32AccessRights = DescAttr.u;
3663	}
3664
3665	/* Validate. */
3666	Assert(!(u16Sel & RT_BIT(2)));
3667	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3668	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3669	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3670	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3671	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3672	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3673	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3674	Assert( (u32Limit & 0xfff) == 0xfff
3675	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3676	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3677	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3678
3679	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3680	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3681	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3682	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3683
3684	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3685	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3686	}
3687
3688	/*
3689	* Guest GDTR.
3690	*/
3691	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3692	{
3693	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3694	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3695
3696	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3697	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3698	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3699	}
3700
3701	/*
3702	* Guest LDTR.
3703	*/
3704	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3705	{
3706	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3707	uint32_t u32Access = 0;
3708	if (!pMixedCtx->ldtr.Attr.u)
3709	u32Access = HMVMX_SEL_UNUSABLE;
3710	else
3711	u32Access = pMixedCtx->ldtr.Attr.u;
3712
3713	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3714	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3715	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3716	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3717
3718	/* Validate. */
3719	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3720	{
3721	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3722	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3723	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3724	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3725	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3726	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3727	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3728	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3729	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3730	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3731	}
3732
3733	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3734	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3735	}
3736
3737	/*
3738	* Guest IDTR.
3739	*/
3740	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3741	{
3742	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3743	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3744
3745	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3746	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3747	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3748	}
3749
3750	return VINF_SUCCESS;
3751	}
3752
3753
3754	/**
3755	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3756	* areas. These MSRs will automatically be loaded to the host CPU on every
3757	* successful VM entry and stored from the host CPU on every successful VM exit.
3758	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3759	*
3760	* @returns VBox status code.
3761	* @param pVCpu Pointer to the VMCPU.
3762	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3763	* out-of-sync. Make sure to update the required fields
3764	* before using them.
3765	*
3766	* @remarks No-long-jump zone!!!
3767	*/
3768	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3769	{
3770	AssertPtr(pVCpu);
3771	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3772
3773	/*
3774	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3775	*/
3776	int rc = VINF_SUCCESS;
3777	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3778	{
3779	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3780	PVM pVM = pVCpu->CTX_SUFF(pVM);
3781	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3782	uint32_t cGuestMsrs = 0;
3783
3784	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3785	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3786	* when the guest really is in 64-bit mode. */
3787	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3788	if (fSupportsLongMode)
3789	{
3790	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3791	pGuestMsr->u32Reserved = 0;
3792	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3793	pGuestMsr++; cGuestMsrs++;
3794	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3795	pGuestMsr->u32Reserved = 0;
3796	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3797	pGuestMsr++; cGuestMsrs++;
3798	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3799	pGuestMsr->u32Reserved = 0;
3800	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3801	pGuestMsr++; cGuestMsrs++;
3802	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3803	pGuestMsr->u32Reserved = 0;
3804	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3805	pGuestMsr++; cGuestMsrs++;
3806	}
3807
3808	/*
3809	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3810	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3811	*/
3812	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3813	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3814	{
3815	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3816	pGuestMsr->u32Reserved = 0;
3817	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3818	AssertRCReturn(rc, rc);
3819	pGuestMsr++; cGuestMsrs++;
3820	}
3821
3822	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3823	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3824	{
3825	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3826	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3827	}
3828
3829	/* Update the VCPU's copy of the guest MSR count. */
3830	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3831	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3832	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3833	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3834
3835	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3836	}
3837
3838	/*
3839	* Guest Sysenter MSRs.
3840	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3841	* VM-exits on WRMSRs for these MSRs.
3842	*/
3843	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3844	{
3845	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3846	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3847	}
3848	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3849	{
3850	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3851	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3852	}
3853	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3854	{
3855	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3856	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3857	}
3858
3859	return rc;
3860	}
3861
3862
3863	/**
3864	* Loads the guest activity state into the guest-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu Pointer to the VMCPU.
3868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3869	* out-of-sync. Make sure to update the required fields
3870	* before using them.
3871	*
3872	* @remarks No-long-jump zone!!!
3873	*/
3874	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3875	{
3876	/** @todo See if we can make use of other states, e.g.
3877	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3878	int rc = VINF_SUCCESS;
3879	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3880	{
3881	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3882	AssertRCReturn(rc, rc);
3883	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3884	}
3885	return rc;
3886	}
3887
3888
3889	/**
3890	* Sets up the appropriate function to run guest code.
3891	*
3892	* @returns VBox status code.
3893	* @param pVCpu Pointer to the VMCPU.
3894	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3895	* out-of-sync. Make sure to update the required fields
3896	* before using them.
3897	*
3898	* @remarks No-long-jump zone!!!
3899	*/
3900	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3901	{
3902	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3903	{
3904	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3905	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3906	#endif
3907	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3908	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3909	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3910	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3911	#else
3912	/* 64-bit host or hybrid host. */
3913	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3914	#endif
3915	}
3916	else
3917	{
3918	/* Guest is not in long mode, use the 32-bit handler. */
3919	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3920	}
3921	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3922	return VINF_SUCCESS;
3923	}
3924
3925
3926	/**
3927	* Wrapper for running the guest code in VT-x.
3928	*
3929	* @returns VBox strict status code.
3930	* @param pVM Pointer to the VM.
3931	* @param pVCpu Pointer to the VMCPU.
3932	* @param pCtx Pointer to the guest-CPU context.
3933	*
3934	* @remarks No-long-jump zone!!!
3935	*/
3936	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3937	{
3938	/*
3939	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3940	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3941	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3942	*/
3943	#ifdef VBOX_WITH_KERNEL_USING_XMM
3944	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3945	#else
3946	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3947	#endif
3948	}
3949
3950
3951	/**
3952	* Report world-switch error and dump some useful debug info.
3953	*
3954	* @param pVM Pointer to the VM.
3955	* @param pVCpu Pointer to the VMCPU.
3956	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3957	* @param pCtx Pointer to the guest-CPU context.
3958	* @param pVmxTransient Pointer to the VMX transient structure (only
3959	* exitReason updated).
3960	*/
3961	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3962	{
3963	Assert(pVM);
3964	Assert(pVCpu);
3965	Assert(pCtx);
3966	Assert(pVmxTransient);
3967	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3968
3969	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3970	switch (rcVMRun)
3971	{
3972	case VERR_VMX_INVALID_VMXON_PTR:
3973	AssertFailed();
3974	break;
3975	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3976	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3977	{
3978	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3979	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3980	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3981	AssertRC(rc);
3982
3983	#ifdef VBOX_STRICT
3984	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3985	pVmxTransient->uExitReason));
3986	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3987	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3988	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3989	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3990	else
3991	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3992
3993	/* VMX control bits. */
3994	uint32_t u32Val;
3995	uint64_t u64Val;
3996	HMVMXHCUINTREG uHCReg;
3997	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3998	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3999	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4000	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4001	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4002	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4003	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4004	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4005	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4006	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4007	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4008	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4009	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4010	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4011	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4012	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4013	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4014	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4015	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4016	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4017	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4018	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4019	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4020	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4021	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4022	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4023	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4024	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4025	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4026	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4027	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4028	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4029	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4030	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4031	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4032	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4033	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4034	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4035	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4036	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4037	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4038	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4039
4040	/* Guest bits. */
4041	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4042	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4043	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4044	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4045	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4046	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4047	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4048	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4049
4050	/* Host bits. */
4051	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4052	Log4(("Host CR0 %#RHr\n", uHCReg));
4053	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4054	Log4(("Host CR3 %#RHr\n", uHCReg));
4055	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4056	Log4(("Host CR4 %#RHr\n", uHCReg));
4057
4058	RTGDTR HostGdtr;
4059	PCX86DESCHC pDesc;
4060	ASMGetGDTR(&HostGdtr);
4061	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4062	Log4(("Host CS %#08x\n", u32Val));
4063	if (u32Val < HostGdtr.cbGdt)
4064	{
4065	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4066	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4067	}
4068
4069	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4070	Log4(("Host DS %#08x\n", u32Val));
4071	if (u32Val < HostGdtr.cbGdt)
4072	{
4073	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4074	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4075	}
4076
4077	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4078	Log4(("Host ES %#08x\n", u32Val));
4079	if (u32Val < HostGdtr.cbGdt)
4080	{
4081	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4082	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4083	}
4084
4085	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4086	Log4(("Host FS %#08x\n", u32Val));
4087	if (u32Val < HostGdtr.cbGdt)
4088	{
4089	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4090	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4091	}
4092
4093	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4094	Log4(("Host GS %#08x\n", u32Val));
4095	if (u32Val < HostGdtr.cbGdt)
4096	{
4097	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4098	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4099	}
4100
4101	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4102	Log4(("Host SS %#08x\n", u32Val));
4103	if (u32Val < HostGdtr.cbGdt)
4104	{
4105	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4106	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4107	}
4108
4109	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4110	Log4(("Host TR %#08x\n", u32Val));
4111	if (u32Val < HostGdtr.cbGdt)
4112	{
4113	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4114	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4115	}
4116
4117	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4118	Log4(("Host TR Base %#RHv\n", uHCReg));
4119	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4120	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4121	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4122	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4123	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4124	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4125	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4126	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4127	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4128	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4129	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4130	Log4(("Host RSP %#RHv\n", uHCReg));
4131	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4132	Log4(("Host RIP %#RHv\n", uHCReg));
4133	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4134	if (HMVMX_IS_64BIT_HOST_MODE())
4135	{
4136	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4137	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4138	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4139	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4140	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4141	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4142	}
4143	# endif
4144	#endif /* VBOX_STRICT */
4145	break;
4146	}
4147
4148	default:
4149	/* Impossible */
4150	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4151	break;
4152	}
4153	NOREF(pVM);
4154	}
4155
4156
4157	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4158	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4159	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4160	#endif
4161	#ifdef VBOX_STRICT
4162	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4163	{
4164	switch (idxField)
4165	{
4166	case VMX_VMCS_GUEST_RIP:
4167	case VMX_VMCS_GUEST_RSP:
4168	case VMX_VMCS_GUEST_SYSENTER_EIP:
4169	case VMX_VMCS_GUEST_SYSENTER_ESP:
4170	case VMX_VMCS_GUEST_GDTR_BASE:
4171	case VMX_VMCS_GUEST_IDTR_BASE:
4172	case VMX_VMCS_GUEST_CS_BASE:
4173	case VMX_VMCS_GUEST_DS_BASE:
4174	case VMX_VMCS_GUEST_ES_BASE:
4175	case VMX_VMCS_GUEST_FS_BASE:
4176	case VMX_VMCS_GUEST_GS_BASE:
4177	case VMX_VMCS_GUEST_SS_BASE:
4178	case VMX_VMCS_GUEST_LDTR_BASE:
4179	case VMX_VMCS_GUEST_TR_BASE:
4180	case VMX_VMCS_GUEST_CR3:
4181	return true;
4182	}
4183	return false;
4184	}
4185
4186	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4187	{
4188	switch (idxField)
4189	{
4190	/* Read-only fields. */
4191	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4192	return true;
4193	}
4194	/* Remaining readable fields should also be writable. */
4195	return hmR0VmxIsValidWriteField(idxField);
4196	}
4197	#endif /* VBOX_STRICT */
4198
4199
4200	/**
4201	* Executes the specified handler in 64-bit mode.
4202	*
4203	* @returns VBox status code.
4204	* @param pVM Pointer to the VM.
4205	* @param pVCpu Pointer to the VMCPU.
4206	* @param pCtx Pointer to the guest CPU context.
4207	* @param enmOp The operation to perform.
4208	* @param cbParam Number of parameters.
4209	* @param paParam Array of 32-bit parameters.
4210	*/
4211	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4212	uint32_t *paParam)
4213	{
4214	int rc, rc2;
4215	PHMGLOBLCPUINFO pCpu;
4216	RTHCPHYS HCPhysCpuPage;
4217	RTCCUINTREG uOldEFlags;
4218
4219	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4220	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4221	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4222	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4223
4224	#ifdef VBOX_STRICT
4225	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4226	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4227
4228	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4229	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4230	#endif
4231
4232	/* Disable interrupts. */
4233	uOldEFlags = ASMIntDisableFlags();
4234
4235	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4236	RTCPUID idHostCpu = RTMpCpuId();
4237	CPUMR0SetLApic(pVM, idHostCpu);
4238	#endif
4239
4240	pCpu = HMR0GetCurrentCpu();
4241	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4242
4243	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4244	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4245
4246	/* Leave VMX Root Mode. */
4247	VMXDisable();
4248
4249	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4250
4251	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4252	CPUMSetHyperEIP(pVCpu, enmOp);
4253	for (int i = (int)cbParam - 1; i >= 0; i--)
4254	CPUMPushHyper(pVCpu, paParam[i]);
4255
4256	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4257
4258	/* Call the switcher. */
4259	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4260	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4261
4262	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4263	/* Make sure the VMX instructions don't cause #UD faults. */
4264	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4265
4266	/* Re-enter VMX Root Mode */
4267	rc2 = VMXEnable(HCPhysCpuPage);
4268	if (RT_FAILURE(rc2))
4269	{
4270	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4271	ASMSetFlags(uOldEFlags);
4272	return rc2;
4273	}
4274
4275	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4276	AssertRC(rc2);
4277	Assert(!(ASMGetFlags() & X86_EFL_IF));
4278	ASMSetFlags(uOldEFlags);
4279	return rc;
4280	}
4281
4282
4283	/**
4284	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4285	* supporting 64-bit guests.
4286	*
4287	* @returns VBox status code.
4288	* @param fResume Whether to VMLAUNCH or VMRESUME.
4289	* @param pCtx Pointer to the guest-CPU context.
4290	* @param pCache Pointer to the VMCS cache.
4291	* @param pVM Pointer to the VM.
4292	* @param pVCpu Pointer to the VMCPU.
4293	*/
4294	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4295	{
4296	uint32_t aParam[6];
4297	PHMGLOBLCPUINFO pCpu = NULL;
4298	RTHCPHYS HCPhysCpuPage = 0;
4299	int rc = VERR_INTERNAL_ERROR_5;
4300
4301	pCpu = HMR0GetCurrentCpu();
4302	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4303
4304	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4305	pCache->uPos = 1;
4306	pCache->interPD = PGMGetInterPaeCR3(pVM);
4307	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4308	#endif
4309
4310	#ifdef VBOX_STRICT
4311	pCache->TestIn.HCPhysCpuPage = 0;
4312	pCache->TestIn.HCPhysVmcs = 0;
4313	pCache->TestIn.pCache = 0;
4314	pCache->TestOut.HCPhysVmcs = 0;
4315	pCache->TestOut.pCache = 0;
4316	pCache->TestOut.pCtx = 0;
4317	pCache->TestOut.eflags = 0;
4318	#endif
4319
4320	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4321	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4322	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4323	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4324	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4325	aParam[5] = 0;
4326
4327	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4328	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4329	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4330	#endif
4331	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4332
4333	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4334	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4335	Assert(pCtx->dr[4] == 10);
4336	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4337	#endif
4338
4339	#ifdef VBOX_STRICT
4340	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4341	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4342	pVCpu->hm.s.vmx.HCPhysVmcs));
4343	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4344	pCache->TestOut.HCPhysVmcs));
4345	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4346	pCache->TestOut.pCache));
4347	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4348	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4349	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4350	pCache->TestOut.pCtx));
4351	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4352	#endif
4353	return rc;
4354	}
4355
4356
4357	/**
4358	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4359	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4360	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4361	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4362	*
4363	* @returns VBox status code.
4364	* @param pVM Pointer to the VM.
4365	* @param pVCpu Pointer to the VMCPU.
4366	*/
4367	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4368	{
4369	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4370	{ \
4371	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4372	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4373	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4374	++cReadFields; \
4375	}
4376
4377	AssertPtr(pVM);
4378	AssertPtr(pVCpu);
4379	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4380	uint32_t cReadFields = 0;
4381
4382	/*
4383	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4384	* and serve to indicate exceptions to the rules.
4385	*/
4386
4387	/* Guest-natural selector base fields. */
4388	#if 0
4389	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4390	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4392	#endif
4393	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4394	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4396	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4397	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4398	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4399	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4400	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4401	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4402	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4404	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4405	#if 0
4406	/* Unused natural width guest-state fields. */
4407	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4408	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4409	#endif
4410	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4412
4413	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4414	#if 0
4415	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4416	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4417	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4418	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4419	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4420	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4421	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4424	#endif
4425
4426	/* Natural width guest-state fields. */
4427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4428	#if 0
4429	/* Currently unused field. */
4430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4431	#endif
4432
4433	if (pVM->hm.s.fNestedPaging)
4434	{
4435	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4436	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4437	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4438	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4439	}
4440	else
4441	{
4442	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4443	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4444	}
4445
4446	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4447	return VINF_SUCCESS;
4448	}
4449
4450
4451	/**
4452	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4453	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4454	* darwin, running 64-bit guests).
4455	*
4456	* @returns VBox status code.
4457	* @param pVCpu Pointer to the VMCPU.
4458	* @param idxField The VMCS field encoding.
4459	* @param u64Val 16, 32 or 64 bits value.
4460	*/
4461	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4462	{
4463	int rc;
4464	switch (idxField)
4465	{
4466	/*
4467	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4468	*/
4469	/* 64-bit Control fields. */
4470	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4471	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4472	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4473	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4474	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4475	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4476	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4477	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4478	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4479	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4480	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4481	case VMX_VMCS64_CTRL_EPTP_FULL:
4482	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4483	/* 64-bit Guest-state fields. */
4484	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4485	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4486	case VMX_VMCS64_GUEST_PAT_FULL:
4487	case VMX_VMCS64_GUEST_EFER_FULL:
4488	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4489	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4490	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4491	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4492	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4493	/* 64-bit Host-state fields. */
4494	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4495	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4496	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4497	{
4498	rc = VMXWriteVmcs32(idxField, u64Val);
4499	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4500	break;
4501	}
4502
4503	/*
4504	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4505	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4506	*/
4507	/* Natural-width Guest-state fields. */
4508	case VMX_VMCS_GUEST_CR3:
4509	case VMX_VMCS_GUEST_ES_BASE:
4510	case VMX_VMCS_GUEST_CS_BASE:
4511	case VMX_VMCS_GUEST_SS_BASE:
4512	case VMX_VMCS_GUEST_DS_BASE:
4513	case VMX_VMCS_GUEST_FS_BASE:
4514	case VMX_VMCS_GUEST_GS_BASE:
4515	case VMX_VMCS_GUEST_LDTR_BASE:
4516	case VMX_VMCS_GUEST_TR_BASE:
4517	case VMX_VMCS_GUEST_GDTR_BASE:
4518	case VMX_VMCS_GUEST_IDTR_BASE:
4519	case VMX_VMCS_GUEST_RSP:
4520	case VMX_VMCS_GUEST_RIP:
4521	case VMX_VMCS_GUEST_SYSENTER_ESP:
4522	case VMX_VMCS_GUEST_SYSENTER_EIP:
4523	{
4524	if (!(u64Val >> 32))
4525	{
4526	/* If this field is 64-bit, VT-x will zero out the top bits. */
4527	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4528	}
4529	else
4530	{
4531	/* Assert that only the 32->64 switcher case should ever come here. */
4532	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4533	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4534	}
4535	break;
4536	}
4537
4538	default:
4539	{
4540	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4541	rc = VERR_INVALID_PARAMETER;
4542	break;
4543	}
4544	}
4545	AssertRCReturn(rc, rc);
4546	return rc;
4547	}
4548
4549
4550	/**
4551	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4552	* hosts (except darwin) for 64-bit guests.
4553	*
4554	* @param pVCpu Pointer to the VMCPU.
4555	* @param idxField The VMCS field encoding.
4556	* @param u64Val 16, 32 or 64 bits value.
4557	*/
4558	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4559	{
4560	AssertPtr(pVCpu);
4561	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4562
4563	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4564	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4565
4566	/* Make sure there are no duplicates. */
4567	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4568	{
4569	if (pCache->Write.aField[i] == idxField)
4570	{
4571	pCache->Write.aFieldVal[i] = u64Val;
4572	return VINF_SUCCESS;
4573	}
4574	}
4575
4576	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4577	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4578	pCache->Write.cValidEntries++;
4579	return VINF_SUCCESS;
4580	}
4581
4582	/* Enable later when the assembly code uses these as callbacks. */
4583	#if 0
4584	/*
4585	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4586	*
4587	* @param pVCpu Pointer to the VMCPU.
4588	* @param pCache Pointer to the VMCS cache.
4589	*
4590	* @remarks No-long-jump zone!!!
4591	*/
4592	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4593	{
4594	AssertPtr(pCache);
4595	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4596	{
4597	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4598	AssertRC(rc);
4599	}
4600	pCache->Write.cValidEntries = 0;
4601	}
4602
4603
4604	/**
4605	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4606	*
4607	* @param pVCpu Pointer to the VMCPU.
4608	* @param pCache Pointer to the VMCS cache.
4609	*
4610	* @remarks No-long-jump zone!!!
4611	*/
4612	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4613	{
4614	AssertPtr(pCache);
4615	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4616	{
4617	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4618	AssertRC(rc);
4619	}
4620	}
4621	#endif
4622	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4623
4624
4625	/**
4626	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4627	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4628	* timer.
4629	*
4630	* @returns VBox status code.
4631	* @param pVCpu Pointer to the VMCPU.
4632	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4633	* out-of-sync. Make sure to update the required fields
4634	* before using them.
4635	* @remarks No-long-jump zone!!!
4636	*/
4637	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4638	{
4639	int rc = VERR_INTERNAL_ERROR_5;
4640	bool fOffsettedTsc = false;
4641	PVM pVM = pVCpu->CTX_SUFF(pVM);
4642	if (pVM->hm.s.vmx.fUsePreemptTimer)
4643	{
4644	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4645
4646	/* Make sure the returned values have sane upper and lower boundaries. */
4647	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4648	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4649	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4650	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4651
4652	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4653	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4654	}
4655	else
4656	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4657
4658	if (fOffsettedTsc)
4659	{
4660	uint64_t u64CurTSC = ASMReadTSC();
4661	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4662	{
4663	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4664	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4665
4666	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4667	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4668	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4669	}
4670	else
4671	{
4672	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4673	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4674	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4675	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4676	}
4677	}
4678	else
4679	{
4680	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4681	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4682	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4683	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4684	}
4685	}
4686
4687
4688	/**
4689	* Determines if an exception is a contributory exception. Contributory
4690	* exceptions are ones which can cause double-faults. Page-fault is
4691	* intentionally not included here as it's a conditional contributory exception.
4692	*
4693	* @returns true if the exception is contributory, false otherwise.
4694	* @param uVector The exception vector.
4695	*/
4696	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4697	{
4698	switch (uVector)
4699	{
4700	case X86_XCPT_GP:
4701	case X86_XCPT_SS:
4702	case X86_XCPT_NP:
4703	case X86_XCPT_TS:
4704	case X86_XCPT_DE:
4705	return true;
4706	default:
4707	break;
4708	}
4709	return false;
4710	}
4711
4712
4713	/**
4714	* Sets an event as a pending event to be injected into the guest.
4715	*
4716	* @param pVCpu Pointer to the VMCPU.
4717	* @param u32IntrInfo The VM-entry interruption-information field.
4718	* @param cbInstr The VM-entry instruction length in bytes (for software
4719	* interrupts, exceptions and privileged software
4720	* exceptions).
4721	* @param u32ErrCode The VM-entry exception error code.
4722	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4723	* page-fault.
4724	*/
4725	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4726	RTGCUINTPTR GCPtrFaultAddress)
4727	{
4728	Assert(!pVCpu->hm.s.Event.fPending);
4729	pVCpu->hm.s.Event.fPending = true;
4730	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4731	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4732	pVCpu->hm.s.Event.cbInstr = cbInstr;
4733	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4734	}
4735
4736
4737	/**
4738	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4739	*
4740	* @param pVCpu Pointer to the VMCPU.
4741	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4742	* out-of-sync. Make sure to update the required fields
4743	* before using them.
4744	*/
4745	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4746	{
4747	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4748	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4749	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4750	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4751	}
4752
4753
4754	/**
4755	* Handle a condition that occurred while delivering an event through the guest
4756	* IDT.
4757	*
4758	* @returns VBox status code (informational error codes included).
4759	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4760	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4761	* continue execution of the guest which will delivery the #DF.
4762	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4763	*
4764	* @param pVCpu Pointer to the VMCPU.
4765	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4766	* out-of-sync. Make sure to update the required fields
4767	* before using them.
4768	* @param pVmxTransient Pointer to the VMX transient structure.
4769	*
4770	* @remarks No-long-jump zone!!!
4771	*/
4772	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4773	{
4774	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4775	AssertRC(rc);
4776	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4777	{
4778	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4779	AssertRCReturn(rc, rc);
4780
4781	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4782	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4783	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4784
4785	typedef enum
4786	{
4787	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4788	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4789	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4790	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4791	} VMXREFLECTXCPT;
4792
4793	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4794	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4795	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4796	{
4797	enmReflect = VMXREFLECTXCPT_XCPT;
4798	#ifdef VBOX_STRICT
4799	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4800	&& uExitVector == X86_XCPT_PF)
4801	{
4802	Log4(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4803	}
4804	#endif
4805	if ( uExitVector == X86_XCPT_PF
4806	&& uIdtVector == X86_XCPT_PF)
4807	{
4808	pVmxTransient->fVectoringPF = true;
4809	Log4(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4810	}
4811	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4812	&& hmR0VmxIsContributoryXcpt(uExitVector)
4813	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4814	\|\| uIdtVector == X86_XCPT_PF))
4815	{
4816	enmReflect = VMXREFLECTXCPT_DF;
4817	}
4818	else if (uIdtVector == X86_XCPT_DF)
4819	enmReflect = VMXREFLECTXCPT_TF;
4820	}
4821	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4822	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4823	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4824	{
4825	/*
4826	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4827	* (whatever they are) as they reoccur when restarting the instruction.
4828	*/
4829	enmReflect = VMXREFLECTXCPT_XCPT;
4830	}
4831
4832	switch (enmReflect)
4833	{
4834	case VMXREFLECTXCPT_XCPT:
4835	{
4836	uint32_t u32ErrCode = 0;
4837	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4838	{
4839	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4840	AssertRCReturn(rc, rc);
4841	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4842	}
4843
4844	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4845	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4846	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4847	rc = VINF_SUCCESS;
4848	Log4(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4849	pVCpu->hm.s.Event.u32ErrCode));
4850	break;
4851	}
4852
4853	case VMXREFLECTXCPT_DF:
4854	{
4855	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4856	rc = VINF_VMX_DOUBLE_FAULT;
4857	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4858	uIdtVector, uExitVector));
4859	break;
4860	}
4861
4862	case VMXREFLECTXCPT_TF:
4863	{
4864	Log4(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4865	rc = VINF_EM_RESET;
4866	break;
4867	}
4868
4869	default:
4870	Assert(rc == VINF_SUCCESS);
4871	break;
4872	}
4873	}
4874	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4875	return rc;
4876	}
4877
4878
4879	/**
4880	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4881	*
4882	* @returns VBox status code.
4883	* @param pVCpu Pointer to the VMCPU.
4884	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4885	* out-of-sync. Make sure to update the required fields
4886	* before using them.
4887	*
4888	* @remarks No-long-jump zone!!!
4889	*/
4890	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4891	{
4892	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4893	{
4894	uint32_t uVal = 0;
4895	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4896	AssertRCReturn(rc, rc);
4897	uint32_t uShadow = 0;
4898	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4899	AssertRCReturn(rc, rc);
4900
4901	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4902	CPUMSetGuestCR0(pVCpu, uVal);
4903	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4904	}
4905	return VINF_SUCCESS;
4906	}
4907
4908
4909	/**
4910	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4911	*
4912	* @returns VBox status code.
4913	* @param pVCpu Pointer to the VMCPU.
4914	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4915	* out-of-sync. Make sure to update the required fields
4916	* before using them.
4917	*
4918	* @remarks No-long-jump zone!!!
4919	*/
4920	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4921	{
4922	int rc = VINF_SUCCESS;
4923	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4924	{
4925	uint32_t uVal = 0;
4926	uint32_t uShadow = 0;
4927	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4928	AssertRCReturn(rc, rc);
4929	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4930	AssertRCReturn(rc, rc);
4931
4932	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4933	CPUMSetGuestCR4(pVCpu, uVal);
4934	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4935	}
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4942	*
4943	* @returns VBox status code.
4944	* @param pVCpu Pointer to the VMCPU.
4945	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4946	* out-of-sync. Make sure to update the required fields
4947	* before using them.
4948	*
4949	* @remarks No-long-jump zone!!!
4950	*/
4951	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4952	{
4953	int rc = VINF_SUCCESS;
4954	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4955	{
4956	uint64_t u64Val = 0;
4957	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4958	AssertRCReturn(rc, rc);
4959
4960	pMixedCtx->rip = u64Val;
4961	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4962	}
4963	return rc;
4964	}
4965
4966
4967	/**
4968	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4969	*
4970	* @returns VBox status code.
4971	* @param pVCpu Pointer to the VMCPU.
4972	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4973	* out-of-sync. Make sure to update the required fields
4974	* before using them.
4975	*
4976	* @remarks No-long-jump zone!!!
4977	*/
4978	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4979	{
4980	int rc = VINF_SUCCESS;
4981	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4982	{
4983	uint64_t u64Val = 0;
4984	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4985	AssertRCReturn(rc, rc);
4986
4987	pMixedCtx->rsp = u64Val;
4988	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4989	}
4990	return rc;
4991	}
4992
4993
4994	/**
4995	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4996	*
4997	* @returns VBox status code.
4998	* @param pVCpu Pointer to the VMCPU.
4999	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5000	* out-of-sync. Make sure to update the required fields
5001	* before using them.
5002	*
5003	* @remarks No-long-jump zone!!!
5004	*/
5005	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5006	{
5007	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5008	{
5009	uint32_t uVal = 0;
5010	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5011	AssertRCReturn(rc, rc);
5012
5013	pMixedCtx->eflags.u32 = uVal;
5014	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5015	{
5016	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5017	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5018
5019	pMixedCtx->eflags.Bits.u1VM = 0;
5020	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5021	}
5022
5023	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5024	}
5025	return VINF_SUCCESS;
5026	}
5027
5028
5029	/**
5030	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5031	* guest-CPU context.
5032	*/
5033	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5034	{
5035	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5036	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5037	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5038	return rc;
5039	}
5040
5041
5042	/**
5043	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5044	* from the guest-state area in the VMCS.
5045	*
5046	* @param pVCpu Pointer to the VMCPU.
5047	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5048	* out-of-sync. Make sure to update the required fields
5049	* before using them.
5050	*
5051	* @remarks No-long-jump zone!!!
5052	*/
5053	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5054	{
5055	uint32_t uIntrState = 0;
5056	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5057	AssertRC(rc);
5058
5059	if (!uIntrState)
5060	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5061	else
5062	{
5063	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5064	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5065	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5066	AssertRC(rc);
5067	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5068	AssertRC(rc);
5069
5070	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5071	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5072	}
5073	}
5074
5075
5076	/**
5077	* Saves the guest's activity state.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu Pointer to the VMCPU.
5081	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5082	* out-of-sync. Make sure to update the required fields
5083	* before using them.
5084	*
5085	* @remarks No-long-jump zone!!!
5086	*/
5087	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5088	{
5089	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5090	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5091	return VINF_SUCCESS;
5092	}
5093
5094
5095	/**
5096	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5097	* the current VMCS into the guest-CPU context.
5098	*
5099	* @returns VBox status code.
5100	* @param pVCpu Pointer to the VMCPU.
5101	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5102	* out-of-sync. Make sure to update the required fields
5103	* before using them.
5104	*
5105	* @remarks No-long-jump zone!!!
5106	*/
5107	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5108	{
5109	int rc = VINF_SUCCESS;
5110	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5111	{
5112	uint32_t u32Val = 0;
5113	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5114	pMixedCtx->SysEnter.cs = u32Val;
5115	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5116	}
5117
5118	uint64_t u64Val = 0;
5119	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5120	{
5121	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5122	pMixedCtx->SysEnter.eip = u64Val;
5123	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5124	}
5125	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5126	{
5127	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5128	pMixedCtx->SysEnter.esp = u64Val;
5129	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5130	}
5131	return rc;
5132	}
5133
5134
5135	/**
5136	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5137	* context.
5138	*
5139	* @returns VBox status code.
5140	* @param pVCpu Pointer to the VMCPU.
5141	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5142	* out-of-sync. Make sure to update the required fields
5143	* before using them.
5144	*
5145	* @remarks No-long-jump zone!!!
5146	*/
5147	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5148	{
5149	int rc = VINF_SUCCESS;
5150	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5151	{
5152	uint64_t u64Val = 0;
5153	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5154	pMixedCtx->fs.u64Base = u64Val;
5155	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5156	}
5157	return rc;
5158	}
5159
5160
5161	/**
5162	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5163	* context.
5164	*
5165	* @returns VBox status code.
5166	* @param pVCpu Pointer to the VMCPU.
5167	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5168	* out-of-sync. Make sure to update the required fields
5169	* before using them.
5170	*
5171	* @remarks No-long-jump zone!!!
5172	*/
5173	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5174	{
5175	int rc = VINF_SUCCESS;
5176	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5177	{
5178	uint64_t u64Val = 0;
5179	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5180	pMixedCtx->gs.u64Base = u64Val;
5181	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5182	}
5183	return rc;
5184	}
5185
5186
5187	/**
5188	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5189	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5190	* and TSC_AUX.
5191	*
5192	* @returns VBox status code.
5193	* @param pVCpu Pointer to the VMCPU.
5194	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5195	* out-of-sync. Make sure to update the required fields
5196	* before using them.
5197	*
5198	* @remarks No-long-jump zone!!!
5199	*/
5200	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5201	{
5202	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5203	return VINF_SUCCESS;
5204
5205	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5206	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5207	{
5208	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5209	pMsr += i;
5210	switch (pMsr->u32IndexMSR)
5211	{
5212	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5213	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5214	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5215	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5216	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5217	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5218	default:
5219	{
5220	AssertFailed();
5221	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5222	}
5223	}
5224	}
5225	#endif
5226
5227	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5228	return VINF_SUCCESS;
5229	}
5230
5231
5232	/**
5233	* Saves the guest control registers from the current VMCS into the guest-CPU
5234	* context.
5235	*
5236	* @returns VBox status code.
5237	* @param pVCpu Pointer to the VMCPU.
5238	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5239	* out-of-sync. Make sure to update the required fields
5240	* before using them.
5241	*
5242	* @remarks No-long-jump zone!!!
5243	*/
5244	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5245	{
5246	/* Guest CR0. Guest FPU. */
5247	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5248	AssertRCReturn(rc, rc);
5249
5250	/* Guest CR4. */
5251	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5252	AssertRCReturn(rc, rc);
5253
5254	/* Guest CR2 - updated always during the world-switch or in #PF. */
5255	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5256	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5257	{
5258	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5259	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5260
5261	PVM pVM = pVCpu->CTX_SUFF(pVM);
5262	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5263	\|\| ( pVM->hm.s.fNestedPaging
5264	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5265	{
5266	uint64_t u64Val = 0;
5267	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5268	if (pMixedCtx->cr3 != u64Val)
5269	{
5270	CPUMSetGuestCR3(pVCpu, u64Val);
5271	if (VMMRZCallRing3IsEnabled(pVCpu))
5272	{
5273	PGMUpdateCR3(pVCpu, u64Val);
5274	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5275	}
5276	else
5277	{
5278	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5279	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5280	}
5281	}
5282
5283	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5284	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5285	{
5286	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5287	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5288	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5289	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5290
5291	if (VMMRZCallRing3IsEnabled(pVCpu))
5292	{
5293	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5294	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5295	}
5296	else
5297	{
5298	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5299	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5300	}
5301	}
5302	}
5303
5304	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5305	}
5306
5307	/*
5308	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5309	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5310	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5311	*
5312	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5313	*/
5314	if (VMMRZCallRing3IsEnabled(pVCpu))
5315	{
5316	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5317	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5318
5319	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5320	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5321
5322	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5323	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5324	}
5325
5326	return rc;
5327	}
5328
5329
5330	/**
5331	* Reads a guest segment register from the current VMCS into the guest-CPU
5332	* context.
5333	*
5334	* @returns VBox status code.
5335	* @param pVCpu Pointer to the VMCPU.
5336	* @param idxSel Index of the selector in the VMCS.
5337	* @param idxLimit Index of the segment limit in the VMCS.
5338	* @param idxBase Index of the segment base in the VMCS.
5339	* @param idxAccess Index of the access rights of the segment in the VMCS.
5340	* @param pSelReg Pointer to the segment selector.
5341	*
5342	* @remarks No-long-jump zone!!!
5343	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5344	* macro as that takes care of whether to read from the VMCS cache or
5345	* not.
5346	*/
5347	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5348	PCPUMSELREG pSelReg)
5349	{
5350	uint32_t u32Val = 0;
5351	int rc = VMXReadVmcs32(idxSel, &u32Val);
5352	AssertRCReturn(rc, rc);
5353	pSelReg->Sel = (uint16_t)u32Val;
5354	pSelReg->ValidSel = (uint16_t)u32Val;
5355	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5356
5357	rc = VMXReadVmcs32(idxLimit, &u32Val);
5358	AssertRCReturn(rc, rc);
5359	pSelReg->u32Limit = u32Val;
5360
5361	uint64_t u64Val = 0;
5362	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5363	AssertRCReturn(rc, rc);
5364	pSelReg->u64Base = u64Val;
5365
5366	rc = VMXReadVmcs32(idxAccess, &u32Val);
5367	AssertRCReturn(rc, rc);
5368	pSelReg->Attr.u = u32Val;
5369
5370	/*
5371	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5372	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5373	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5374	*
5375	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5376	*/
5377	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5378	{
5379	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5380	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5381	}
5382	return VINF_SUCCESS;
5383	}
5384
5385
5386	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5387	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5388	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5389	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5390	#else
5391	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5392	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5393	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5394	#endif
5395
5396
5397	/**
5398	* Saves the guest segment registers from the current VMCS into the guest-CPU
5399	* context.
5400	*
5401	* @returns VBox status code.
5402	* @param pVCpu Pointer to the VMCPU.
5403	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5404	* out-of-sync. Make sure to update the required fields
5405	* before using them.
5406	*
5407	* @remarks No-long-jump zone!!!
5408	*/
5409	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5410	{
5411	/* Guest segment registers. */
5412	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5413	{
5414	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5415	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5416	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5417	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5418	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5419	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5420	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5421
5422	/* Restore segment attributes for real-on-v86 mode hack. */
5423	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5424	{
5425	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5426	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5427	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5428	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5429	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5430	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5431	}
5432	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5433	}
5434
5435	return VINF_SUCCESS;
5436	}
5437
5438
5439	/**
5440	* Saves the guest descriptor table registers and task register from the current
5441	* VMCS into the guest-CPU context.
5442	*
5443	* @returns VBox status code.
5444	* @param pVCpu Pointer to the VMCPU.
5445	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5446	* out-of-sync. Make sure to update the required fields
5447	* before using them.
5448	*
5449	* @remarks No-long-jump zone!!!
5450	*/
5451	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5452	{
5453	int rc = VINF_SUCCESS;
5454
5455	/* Guest LDTR. */
5456	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5457	{
5458	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5459	AssertRCReturn(rc, rc);
5460	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5461	}
5462
5463	/* Guest GDTR. */
5464	uint64_t u64Val = 0;
5465	uint32_t u32Val = 0;
5466	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5467	{
5468	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5469	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5470	pMixedCtx->gdtr.pGdt = u64Val;
5471	pMixedCtx->gdtr.cbGdt = u32Val;
5472	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5473	}
5474
5475	/* Guest IDTR. */
5476	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5477	{
5478	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5479	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5480	pMixedCtx->idtr.pIdt = u64Val;
5481	pMixedCtx->idtr.cbIdt = u32Val;
5482	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5483	}
5484
5485	/* Guest TR. */
5486	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5487	{
5488	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5489	AssertRCReturn(rc, rc);
5490
5491	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5492	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5493	{
5494	rc = VMXLOCAL_READ_SEG(TR, tr);
5495	AssertRCReturn(rc, rc);
5496	}
5497	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5498	}
5499	return rc;
5500	}
5501
5502	#undef VMXLOCAL_READ_SEG
5503
5504
5505	/**
5506	* Saves the guest debug registers from the current VMCS into the guest-CPU
5507	* context.
5508	*
5509	* @returns VBox status code.
5510	* @param pVCpu Pointer to the VMCPU.
5511	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5512	* out-of-sync. Make sure to update the required fields
5513	* before using them.
5514	*
5515	* @remarks No-long-jump zone!!!
5516	*/
5517	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5518	{
5519	int rc = VINF_SUCCESS;
5520	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5521	{
5522	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5523	uint32_t u32Val;
5524	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5525	pMixedCtx->dr[7] = u32Val;
5526
5527	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5528	}
5529	return rc;
5530	}
5531
5532
5533	/**
5534	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5535	*
5536	* @returns VBox status code.
5537	* @param pVCpu Pointer to the VMCPU.
5538	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5539	* out-of-sync. Make sure to update the required fields
5540	* before using them.
5541	*
5542	* @remarks No-long-jump zone!!!
5543	*/
5544	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5545	{
5546	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5547	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5548	return VINF_SUCCESS;
5549	}
5550
5551
5552	/**
5553	* Saves the entire guest state from the currently active VMCS into the
5554	* guest-CPU context. This essentially VMREADs all guest-data.
5555	*
5556	* @returns VBox status code.
5557	* @param pVCpu Pointer to the VMCPU.
5558	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5559	* out-of-sync. Make sure to update the required fields
5560	* before using them.
5561	*/
5562	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5563	{
5564	Assert(pVCpu);
5565	Assert(pMixedCtx);
5566
5567	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5568	return VINF_SUCCESS;
5569
5570	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5571	there is no real need to. */
5572	if (VMMRZCallRing3IsEnabled(pVCpu))
5573	VMMR0LogFlushDisable(pVCpu);
5574	else
5575	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5576	Log4Func(("\n"));
5577
5578	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5579	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5580
5581	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5582	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5583
5584	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5585	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5586
5587	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5588	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5589
5590	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5591	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5592
5593	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5594	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5595
5596	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5597	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5598
5599	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5600	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5601
5602	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5603	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5604
5605	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5606	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5607
5608	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5609	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5610
5611	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5612	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5613
5614	if (VMMRZCallRing3IsEnabled(pVCpu))
5615	VMMR0LogFlushEnable(pVCpu);
5616
5617	return rc;
5618	}
5619
5620
5621	/**
5622	* Check per-VM and per-VCPU force flag actions that require us to go back to
5623	* ring-3 for one reason or another.
5624	*
5625	* @returns VBox status code (information status code included).
5626	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5627	* ring-3.
5628	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5629	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5630	* interrupts)
5631	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5632	* all EMTs to be in ring-3.
5633	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5634	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5635	* to the EM loop.
5636	*
5637	* @param pVM Pointer to the VM.
5638	* @param pVCpu Pointer to the VMCPU.
5639	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5640	* out-of-sync. Make sure to update the required fields
5641	* before using them.
5642	*/
5643	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5644	{
5645	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5646
5647	int rc = VERR_INTERNAL_ERROR_5;
5648	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5649	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5650	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5651	{
5652	/* We need the control registers now, make sure the guest-CPU context is updated. */
5653	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5654	AssertRCReturn(rc, rc);
5655
5656	/* Pending HM CR3 sync. */
5657	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5658	{
5659	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5660	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5661	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5662	}
5663
5664	/* Pending HM PAE PDPEs. */
5665	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5666	{
5667	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5668	AssertRC(rc);
5669	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5670	}
5671
5672	/* Pending PGM C3 sync. */
5673	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5674	{
5675	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5676	if (rc != VINF_SUCCESS)
5677	{
5678	AssertRC(rc);
5679	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5680	return rc;
5681	}
5682	}
5683
5684	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5685	/* -XXX- what was that about single stepping? */
5686	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5687	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5688	{
5689	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5690	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5691	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5692	return rc;
5693	}
5694
5695	/* Pending VM request packets, such as hardware interrupts. */
5696	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5697	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5698	{
5699	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5700	return VINF_EM_PENDING_REQUEST;
5701	}
5702
5703	/* Pending PGM pool flushes. */
5704	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5705	{
5706	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5707	return VINF_PGM_POOL_FLUSH_PENDING;
5708	}
5709
5710	/* Pending DMA requests. */
5711	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5712	{
5713	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5714	return VINF_EM_RAW_TO_R3;
5715	}
5716	}
5717
5718	/* Paranoia. */
5719	Assert(rc != VERR_EM_INTERPRETER);
5720	return VINF_SUCCESS;
5721	}
5722
5723
5724	/**
5725	* Converts any TRPM trap into a pending VMX event. This is typically used when
5726	* entering from ring-3 (not longjmp returns).
5727	*
5728	* @param pVCpu Pointer to the VMCPU.
5729	*/
5730	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5731	{
5732	Assert(TRPMHasTrap(pVCpu));
5733	Assert(!pVCpu->hm.s.Event.fPending);
5734
5735	uint8_t uVector;
5736	TRPMEVENT enmTrpmEvent;
5737	RTGCUINT uErrCode;
5738	RTGCUINTPTR GCPtrFaultAddress;
5739	uint8_t cbInstr;
5740
5741	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5742	AssertRC(rc);
5743
5744	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5745	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5746	if (enmTrpmEvent == TRPM_TRAP)
5747	{
5748	switch (uVector)
5749	{
5750	case X86_XCPT_BP:
5751	case X86_XCPT_OF:
5752	{
5753	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5754	break;
5755	}
5756
5757	case X86_XCPT_PF:
5758	case X86_XCPT_DF:
5759	case X86_XCPT_TS:
5760	case X86_XCPT_NP:
5761	case X86_XCPT_SS:
5762	case X86_XCPT_GP:
5763	case X86_XCPT_AC:
5764	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5765	/* no break! */
5766	default:
5767	{
5768	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5769	break;
5770	}
5771	}
5772	}
5773	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5774	{
5775	if (uVector == X86_XCPT_NMI)
5776	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5777	else
5778	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5779	}
5780	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5781	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5782	else
5783	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5784
5785	rc = TRPMResetTrap(pVCpu);
5786	AssertRC(rc);
5787	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5788	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5789	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5790	}
5791
5792
5793	/**
5794	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5795	* VT-x to execute any instruction.
5796	*
5797	* @param pvCpu Pointer to the VMCPU.
5798	*/
5799	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5800	{
5801	Assert(pVCpu->hm.s.Event.fPending);
5802
5803	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5804	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5805	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5806	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5807
5808	/* If a trap was already pending, we did something wrong! */
5809	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5810
5811	TRPMEVENT enmTrapType;
5812	switch (uVectorType)
5813	{
5814	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5815	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5816	enmTrapType = TRPM_HARDWARE_INT;
5817	break;
5818	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5819	enmTrapType = TRPM_SOFTWARE_INT;
5820	break;
5821	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5822	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5823	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5824	enmTrapType = TRPM_TRAP;
5825	break;
5826	default:
5827	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5828	enmTrapType = TRPM_32BIT_HACK;
5829	break;
5830	}
5831
5832	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5833
5834	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5835	AssertRC(rc);
5836
5837	if (fErrorCodeValid)
5838	TRPMSetErrorCode(pVCpu, uErrorCode);
5839
5840	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5841	&& uVector == X86_XCPT_PF)
5842	{
5843	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5844	}
5845	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5846	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5847	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5848	{
5849	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5850	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5851	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5852	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5853	}
5854	pVCpu->hm.s.Event.fPending = false;
5855	}
5856
5857
5858	/**
5859	* Does the necessary state syncing before doing a longjmp to ring-3.
5860	*
5861	* @param pVM Pointer to the VM.
5862	* @param pVCpu Pointer to the VMCPU.
5863	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5864	* out-of-sync. Make sure to update the required fields
5865	* before using them.
5866	* @param rcExit The reason for exiting to ring-3. Can be
5867	* VINF_VMM_UNKNOWN_RING3_CALL.
5868	*
5869	* @remarks No-long-jmp zone!!!
5870	*/
5871	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5872	{
5873	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5874	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5875
5876	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5877	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5878	AssertRC(rc);
5879
5880	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5881	if (CPUMIsGuestFPUStateActive(pVCpu))
5882	{
5883	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5884	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5885	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5886	}
5887
5888	/* Restore debug registers if necessary and resync on next R0 reentry. */
5889	if (CPUMIsGuestDebugStateActive(pVCpu))
5890	{
5891	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5892	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5893	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5894	}
5895	else if (CPUMIsHyperDebugStateActive(pVCpu))
5896	{
5897	CPUMR0LoadHostDebugState(pVM, pVCpu);
5898	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5899	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5900	}
5901
5902	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5903	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
5904	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5905	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5906	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5907	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5908	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5909	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5910	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5911	}
5912
5913
5914	/**
5915	* An action requires us to go back to ring-3. This function does the necessary
5916	* steps before we can safely return to ring-3. This is not the same as longjmps
5917	* to ring-3, this is voluntary.
5918	*
5919	* @param pVM Pointer to the VM.
5920	* @param pVCpu Pointer to the VMCPU.
5921	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5922	* out-of-sync. Make sure to update the required fields
5923	* before using them.
5924	* @param rcExit The reason for exiting to ring-3. Can be
5925	* VINF_VMM_UNKNOWN_RING3_CALL.
5926	*/
5927	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5928	{
5929	Assert(pVM);
5930	Assert(pVCpu);
5931	Assert(pMixedCtx);
5932	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5933
5934	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5935	{
5936	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5937	return;
5938	}
5939	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5940	{
5941	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5942	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5943	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5944	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5945	return;
5946	}
5947
5948	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5949	VMMRZCallRing3Disable(pVCpu);
5950	Log4(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5951
5952	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5953	if (pVCpu->hm.s.Event.fPending)
5954	{
5955	hmR0VmxPendingEventToTrpmTrap(pVCpu);
5956	Assert(!pVCpu->hm.s.Event.fPending);
5957	}
5958
5959	/* Sync. the guest state. */
5960	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5961	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5962
5963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5964	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5965	\| CPUM_CHANGED_LDTR
5966	\| CPUM_CHANGED_GDTR
5967	\| CPUM_CHANGED_IDTR
5968	\| CPUM_CHANGED_TR
5969	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5970
5971	/* On our way back from ring-3 the following needs to be done. */
5972	/** @todo This can change with preemption hooks. */
5973	if (rcExit == VINF_EM_RAW_INTERRUPT)
5974	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5975	else
5976	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5977
5978	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5979	VMMRZCallRing3Enable(pVCpu);
5980	}
5981
5982
5983	/**
5984	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5985	* longjump to ring-3 and possibly get preempted.
5986	*
5987	* @param pVCpu Pointer to the VMCPU.
5988	* @param enmOperation The operation causing the ring-3 longjump.
5989	* @param pvUser The user argument (pointer to the possibly
5990	* out-of-date guest-CPU context).
5991	*
5992	* @remarks Must never be called with @a enmOperation ==
5993	* VMMCALLRING3_VM_R0_ASSERTION.
5994	*/
5995	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5996	{
5997	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5998	Assert(pVCpu);
5999	Assert(pvUser);
6000	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6002
6003	VMMRZCallRing3Disable(pVCpu);
6004	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6005	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
6006	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6007	VMMRZCallRing3Enable(pVCpu);
6008	}
6009
6010
6011	/**
6012	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6013	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6014	*
6015	* @param pVCpu Pointer to the VMCPU.
6016	*/
6017	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6018	{
6019	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6020	{
6021	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6022	{
6023	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6024	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6025	AssertRC(rc);
6026	}
6027	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6028	}
6029
6030
6031	/**
6032	* Injects any pending events into the guest if the guest is in a state to
6033	* receive them.
6034	*
6035	* @returns VBox status code (informational status codes included).
6036	* @param pVCpu Pointer to the VMCPU.
6037	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6038	* out-of-sync. Make sure to update the required fields
6039	* before using them.
6040	*/
6041	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6042	{
6043	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6044	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6045	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6046	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6047
6048	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6049	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6050	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6051	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6052	Assert(!TRPMHasTrap(pVCpu));
6053
6054	int rc = VINF_SUCCESS;
6055	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6056	{
6057	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6058	bool fInject = true;
6059	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6060	{
6061	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6062	AssertRCReturn(rc, rc);
6063	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6064	if ( fBlockInt
6065	\|\| fBlockSti
6066	\|\| fBlockMovSS)
6067	{
6068	fInject = false;
6069	}
6070	}
6071	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6072	&& ( fBlockMovSS
6073	\|\| fBlockSti))
6074	{
6075	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6076	fInject = false;
6077	}
6078
6079	if (fInject)
6080	{
6081	Log4(("Injecting pending event\n"));
6082	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6083	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6084	AssertRCReturn(rc, rc);
6085	pVCpu->hm.s.Event.fPending = false;
6086	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
6087	}
6088	else
6089	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6090	} /** @todo SMI. SMIs take priority over NMIs. */
6091	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6092	{
6093	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6094	if ( !fBlockMovSS
6095	&& !fBlockSti)
6096	{
6097	Log4(("Injecting NMI\n"));
6098	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6099	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6100	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6101	0 /* GCPtrFaultAddress */, &uIntrState);
6102	AssertRCReturn(rc, rc);
6103	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6104	}
6105	else
6106	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6107	}
6108	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6109	{
6110	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6111	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6112	AssertRCReturn(rc, rc);
6113	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6114	if ( !fBlockInt
6115	&& !fBlockSti
6116	&& !fBlockMovSS)
6117	{
6118	uint8_t u8Interrupt;
6119	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6120	if (RT_SUCCESS(rc))
6121	{
6122	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6123	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6124	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6125	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6126	0 /* GCPtrFaultAddress */, &uIntrState);
6127	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6128	}
6129	else
6130	{
6131	/** @todo Does this actually happen? If not turn it into an assertion. */
6132	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6133	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6134	rc = VINF_SUCCESS;
6135	}
6136	}
6137	else
6138	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6139	}
6140
6141	/*
6142	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6143	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6144	*/
6145	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6146	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6147	int rc2 = VINF_SUCCESS;
6148	if ( fBlockSti
6149	\|\| fBlockMovSS)
6150	{
6151	if (!DBGFIsStepping(pVCpu))
6152	{
6153	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6154	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6155	{
6156	/*
6157	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6158	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6159	*/
6160	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6161	AssertRCReturn(rc, rc);
6162	}
6163	}
6164	else
6165	{
6166	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6167	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6168	uIntrState = 0;
6169	}
6170	}
6171
6172	/*
6173	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6174	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6175	*/
6176	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6177	AssertRC(rc2);
6178
6179	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6180	return rc;
6181	}
6182
6183
6184	/**
6185	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6186	*
6187	* @param pVCpu Pointer to the VMCPU.
6188	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6189	* out-of-sync. Make sure to update the required fields
6190	* before using them.
6191	*/
6192	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6193	{
6194	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6195	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6196	}
6197
6198
6199	/**
6200	* Injects a double-fault (#DF) exception into the VM.
6201	*
6202	* @returns VBox status code (informational status code included).
6203	* @param pVCpu Pointer to the VMCPU.
6204	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6205	* out-of-sync. Make sure to update the required fields
6206	* before using them.
6207	*/
6208	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6209	{
6210	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6211	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6212	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6213	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6214	puIntrState);
6215	}
6216
6217
6218	/**
6219	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6220	*
6221	* @param pVCpu Pointer to the VMCPU.
6222	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6223	* out-of-sync. Make sure to update the required fields
6224	* before using them.
6225	*/
6226	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6227	{
6228	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6229	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6230	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6231	}
6232
6233
6234	/**
6235	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6236	*
6237	* @param pVCpu Pointer to the VMCPU.
6238	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6239	* out-of-sync. Make sure to update the required fields
6240	* before using them.
6241	* @param cbInstr The value of RIP that is to be pushed on the guest
6242	* stack.
6243	*/
6244	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6245	{
6246	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6247	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6248	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6249	}
6250
6251
6252	/**
6253	* Injects a general-protection (#GP) fault into the VM.
6254	*
6255	* @returns VBox status code (informational status code included).
6256	* @param pVCpu Pointer to the VMCPU.
6257	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6258	* out-of-sync. Make sure to update the required fields
6259	* before using them.
6260	* @param u32ErrorCode The error code associated with the #GP.
6261	*/
6262	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6263	uint32_t *puIntrState)
6264	{
6265	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6266	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6267	if (fErrorCodeValid)
6268	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6269	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6270	puIntrState);
6271	}
6272
6273
6274	/**
6275	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6276	*
6277	* @param pVCpu Pointer to the VMCPU.
6278	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6279	* out-of-sync. Make sure to update the required fields
6280	* before using them.
6281	* @param uVector The software interrupt vector number.
6282	* @param cbInstr The value of RIP that is to be pushed on the guest
6283	* stack.
6284	*/
6285	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6286	{
6287	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6288	if ( uVector == X86_XCPT_BP
6289	\|\| uVector == X86_XCPT_OF)
6290	{
6291	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6292	}
6293	else
6294	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6295	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6296	}
6297
6298
6299	/**
6300	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6301	* stack.
6302	*
6303	* @returns VBox status code (information status code included).
6304	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6305	* @param pVM Pointer to the VM.
6306	* @param pMixedCtx Pointer to the guest-CPU context.
6307	* @param uValue The value to push to the guest stack.
6308	*/
6309	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6310	{
6311	/*
6312	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6313	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6314	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6315	*/
6316	if (pMixedCtx->sp == 1)
6317	return VINF_EM_RESET;
6318	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6319	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6320	AssertRCReturn(rc, rc);
6321	return rc;
6322	}
6323
6324
6325	/**
6326	* Injects an event into the guest upon VM-entry by updating the relevant fields
6327	* in the VM-entry area in the VMCS.
6328	*
6329	* @returns VBox status code (informational error codes included).
6330	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6331	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6332	*
6333	* @param pVCpu Pointer to the VMCPU.
6334	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6335	* be out-of-sync. Make sure to update the required
6336	* fields before using them.
6337	* @param u64IntrInfo The VM-entry interruption-information field.
6338	* @param cbInstr The VM-entry instruction length in bytes (for
6339	* software interrupts, exceptions and privileged
6340	* software exceptions).
6341	* @param u32ErrCode The VM-entry exception error code.
6342	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6343	* @param puIntrState Pointer to the current guest interruptibility-state.
6344	* This interruptibility-state will be updated if
6345	* necessary. This cannot not be NULL.
6346	*
6347	* @remarks No-long-jump zone!!!
6348	* @remarks Requires CR0!
6349	*/
6350	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6351	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6352	{
6353	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6354	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6355	Assert(puIntrState);
6356	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6357
6358	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6359	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6360
6361	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6362	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6363	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6364
6365	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6366
6367	/* We require CR0 to check if the guest is in real-mode. */
6368	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6369	AssertRCReturn(rc, rc);
6370
6371	/*
6372	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6373	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6374	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6375	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6376	*/
6377	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6378	{
6379	PVM pVM = pVCpu->CTX_SUFF(pVM);
6380	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6381	{
6382	Assert(PDMVmmDevHeapIsEnabled(pVM));
6383	Assert(pVM->hm.s.vmx.pRealModeTSS);
6384
6385	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6386	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6387	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6388	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6389	AssertRCReturn(rc, rc);
6390	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6391
6392	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6393	const size_t cbIdtEntry = 4;
6394	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6395	{
6396	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6397	if (uVector == X86_XCPT_DF)
6398	return VINF_EM_RESET;
6399	else if (uVector == X86_XCPT_GP)
6400	{
6401	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6402	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6403	}
6404
6405	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6406	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6407	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6408	}
6409
6410	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6411	uint16_t uGuestIp = pMixedCtx->ip;
6412	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6413	{
6414	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6415	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6416	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6417	}
6418	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6419	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6420
6421	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6422	uint16_t offIdtEntry = 0;
6423	RTSEL selIdtEntry = 0;
6424	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6425	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6426	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6427	AssertRCReturn(rc, rc);
6428
6429	/* Construct the stack frame for the interrupt/exception handler. */
6430	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6431	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6432	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6433	AssertRCReturn(rc, rc);
6434
6435	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6436	if (rc == VINF_SUCCESS)
6437	{
6438	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6439	pMixedCtx->rip = offIdtEntry;
6440	pMixedCtx->cs.Sel = selIdtEntry;
6441	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6442	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6443	&& uVector == X86_XCPT_PF)
6444	{
6445	pMixedCtx->cr2 = GCPtrFaultAddress;
6446	}
6447	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6448	\| HM_CHANGED_GUEST_RIP
6449	\| HM_CHANGED_GUEST_RFLAGS
6450	\| HM_CHANGED_GUEST_RSP;
6451
6452	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6453	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6454	{
6455	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6456	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6457	Log4(("Clearing inhibition due to STI.\n"));
6458	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6459	}
6460	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6461	}
6462	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6463	return rc;
6464	}
6465	else
6466	{
6467	/*
6468	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6469	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6470	*/
6471	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6472	}
6473	}
6474
6475	/* Validate. */
6476	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6477	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6478	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6479
6480	/* Inject. */
6481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6482	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6483	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6484	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6485
6486	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6487	&& uVector == X86_XCPT_PF)
6488	{
6489	pMixedCtx->cr2 = GCPtrFaultAddress;
6490	}
6491
6492	Log4(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr,
6493	pMixedCtx->cr2));
6494
6495	AssertRCReturn(rc, rc);
6496	return rc;
6497	}
6498
6499
6500	/**
6501	* Enters the VT-x session.
6502	*
6503	* @returns VBox status code.
6504	* @param pVM Pointer to the VM.
6505	* @param pVCpu Pointer to the VMCPU.
6506	* @param pCpu Pointer to the CPU info struct.
6507	*/
6508	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6509	{
6510	AssertPtr(pVM);
6511	AssertPtr(pVCpu);
6512	Assert(pVM->hm.s.vmx.fSupported);
6513	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6514	NOREF(pCpu);
6515
6516	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6517
6518	/* Make sure we're in VMX root mode. */
6519	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6520	if (!(u32HostCR4 & X86_CR4_VMXE))
6521	{
6522	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6523	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6524	}
6525
6526	/* Load the active VMCS as the current one. */
6527	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6528	if (RT_FAILURE(rc))
6529	return rc;
6530
6531	/** @todo this will change with preemption hooks where can can VMRESUME as long
6532	* as we're no preempted. */
6533	pVCpu->hm.s.fResumeVM = false;
6534	return VINF_SUCCESS;
6535	}
6536
6537
6538	/**
6539	* Leaves the VT-x session.
6540	*
6541	* @returns VBox status code.
6542	* @param pVM Pointer to the VM.
6543	* @param pVCpu Pointer to the VMCPU.
6544	* @param pCtx Pointer to the guest-CPU context.
6545	*/
6546	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6547	{
6548	AssertPtr(pVCpu);
6549	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6550	NOREF(pVM);
6551	NOREF(pCtx);
6552
6553	/** @todo this will change with preemption hooks where we only VMCLEAR when
6554	* we are actually going to be preempted, not all the time like we
6555	* currently do. */
6556
6557	/* Restore host-state bits that VT-x only restores partially. */
6558	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6559	{
6560	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6561	/** @todo r=ramshankar: This is broken when
6562	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6563	* VMXRestoreHostState() may unconditionally enables interrupts. */
6564	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6565	#else
6566	Assert(ASMIntAreEnabled());
6567	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6568	#endif
6569	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6570	}
6571
6572	/*
6573	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6574	* and mark the VMCS launch-state as "clear".
6575	*/
6576	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6577	return rc;
6578	}
6579
6580
6581	/**
6582	* Saves the host state in the VMCS host-state.
6583	* Sets up the VM-exit MSR-load area.
6584	*
6585	* The CPU state will be loaded from these fields on every successful VM-exit.
6586	*
6587	* @returns VBox status code.
6588	* @param pVM Pointer to the VM.
6589	* @param pVCpu Pointer to the VMCPU.
6590	*
6591	* @remarks No-long-jump zone!!!
6592	*/
6593	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6594	{
6595	AssertPtr(pVM);
6596	AssertPtr(pVCpu);
6597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6598
6599	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6600
6601	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6602	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6603	return VINF_SUCCESS;
6604
6605	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6606	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6607
6608	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6609	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6610
6611	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6612	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6613
6614	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6615	return rc;
6616	}
6617
6618
6619	/**
6620	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6621	* loaded from these fields on every successful VM-entry.
6622	*
6623	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6624	* Sets up the VM-entry controls.
6625	* Sets up the appropriate VMX non-root function to execute guest code based on
6626	* the guest CPU mode.
6627	*
6628	* @returns VBox status code.
6629	* @param pVM Pointer to the VM.
6630	* @param pVCpu Pointer to the VMCPU.
6631	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6632	* out-of-sync. Make sure to update the required fields
6633	* before using them.
6634	*
6635	* @remarks No-long-jump zone!!!
6636	*/
6637	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6638	{
6639	AssertPtr(pVM);
6640	AssertPtr(pVCpu);
6641	AssertPtr(pMixedCtx);
6642	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6643
6644	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6645
6646	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6647
6648	/* Determine real-on-v86 mode. */
6649	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6650	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6651	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6652	{
6653	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6654	}
6655
6656	/*
6657	* Load the guest-state into the VMCS.
6658	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6659	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6660	*/
6661	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6662	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6663
6664	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6665	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6666
6667	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6668	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6669
6670	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6671	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6672
6673	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6674	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6675	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6676
6677	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6678	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6679
6680	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6681	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6682
6683	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6684	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6685
6686	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6687	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6688	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6689
6690	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6691	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6692
6693	/* Clear any unused and reserved bits. */
6694	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_GUEST_CR2
6695	\| HM_CHANGED_VMX_RESERVED1
6696	\| HM_CHANGED_VMX_RESERVED2);
6697
6698	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6699	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6700	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6701
6702	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6703	return rc;
6704	}
6705
6706
6707	/**
6708	* Does the preparations before executing guest code in VT-x.
6709	*
6710	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6711	* recompiler. We must be cautious what we do here regarding committing
6712	* guest-state information into the the VMCS assuming we assuredly execute the
6713	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6714	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6715	* that the recompiler can (and should) use them when it resumes guest
6716	* execution. Otherwise such operations must be done when we can no longer
6717	* exit to ring-3.
6718	*
6719	* @returns VBox status code (informational status codes included).
6720	* @retval VINF_SUCCESS if we can proceed with running the guest.
6721	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6722	* into the guest.
6723	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6724	*
6725	* @param pVM Pointer to the VM.
6726	* @param pVCpu Pointer to the VMCPU.
6727	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6728	* out-of-sync. Make sure to update the required fields
6729	* before using them.
6730	* @param pVmxTransient Pointer to the VMX transient structure.
6731	*
6732	* @remarks Called with preemption disabled.
6733	*/
6734	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6735	{
6736	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6737
6738	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6739	PGMRZDynMapFlushAutoSet(pVCpu);
6740	#endif
6741
6742	/* Check force flag actions that might require us to go back to ring-3. */
6743	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6744	if (rc != VINF_SUCCESS)
6745	return rc;
6746
6747	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6748	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6749	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6750	{
6751	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6752	RTGCPHYS GCPhysApicBase;
6753	GCPhysApicBase = pMixedCtx->msrApicBase;
6754	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6755
6756	/* Unalias any existing mapping. */
6757	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6758	AssertRCReturn(rc, rc);
6759
6760	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6761	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6762	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6763	AssertRCReturn(rc, rc);
6764
6765	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6766	}
6767
6768	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6769	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6770	pVmxTransient->uEFlags = ASMIntDisableFlags();
6771	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6772	{
6773	ASMSetFlags(pVmxTransient->uEFlags);
6774	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6775	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6776	return VINF_EM_RAW_INTERRUPT;
6777	}
6778	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6779	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6780	#endif
6781
6782	/*
6783	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6784	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6785	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6786	*/
6787	/** @todo Rework event evaluation and injection to be completely separate. */
6788	if (TRPMHasTrap(pVCpu))
6789	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6790
6791	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6792	AssertRCReturn(rc, rc);
6793	return rc;
6794	}
6795
6796
6797	/**
6798	* Prepares to run guest code in VT-x and we've committed to doing so. This
6799	* means there is no backing out to ring-3 or anywhere else at this
6800	* point.
6801	*
6802	* @param pVM Pointer to the VM.
6803	* @param pVCpu Pointer to the VMCPU.
6804	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6805	* out-of-sync. Make sure to update the required fields
6806	* before using them.
6807	* @param pVmxTransient Pointer to the VMX transient structure.
6808	*
6809	* @remarks Called with preemption disabled.
6810	* @remarks No-long-jump zone!!!
6811	*/
6812	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6813	{
6814	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6815	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6816
6817	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6818	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6819	pVmxTransient->uEFlags = ASMIntDisableFlags();
6820	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6821	#endif
6822
6823	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6824	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6825	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6826	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6827	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6828	#endif
6829	int rc = VINF_SUCCESS;
6830	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6831	{
6832	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6833	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6834	}
6835	else if (pVCpu->hm.s.fContextUseFlags)
6836	{
6837	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6838	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6839	}
6840	AssertRC(rc);
6841	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6842
6843	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6844	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6845	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6846
6847	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6848	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6849	{
6850	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6851	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6852	}
6853
6854	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6855	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6856	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6857
6858	/*
6859	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6860	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6861	*/
6862	if (pVM->hm.s.fTPRPatchingActive)
6863	{
6864	Assert(!CPUMIsGuestInLongMode(pVCpu));
6865
6866	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6867	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6868	AssertRC(rc);
6869
6870	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6871	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6872	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6873	see hmR0VmxLoadGuestApicState(). */
6874	}
6875
6876	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6877	/*
6878	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6879	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6880	*/
6881	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6882	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6883	{
6884	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6885	uint64_t u64HostTscAux = 0;
6886	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6887	AssertRC(rc2);
6888	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6889	}
6890	#endif
6891
6892	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6893	to start executing. */
6894	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6895	}
6896
6897
6898	/**
6899	* Performs some essential restoration of state after running guest code in
6900	* VT-x.
6901	*
6902	* @param pVM Pointer to the VM.
6903	* @param pVCpu Pointer to the VMCPU.
6904	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6905	* out-of-sync. Make sure to update the required fields
6906	* before using them.
6907	* @param pVmxTransient Pointer to the VMX transient structure.
6908	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6909	*
6910	* @remarks Called with interrupts disabled.
6911	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6912	* unconditionally when it is safe to do so.
6913	*/
6914	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6915	{
6916	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6917	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6918
6919	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6920	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6921	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6922	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6923	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6924
6925	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6926	{
6927	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6928	/* Restore host's TSC_AUX. */
6929	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6930	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6931	#endif
6932	/** @todo Find a way to fix hardcoding a guestimate. */
6933	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6934	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6935	}
6936
6937	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6938	Assert(!(ASMGetFlags() & X86_EFL_IF));
6939	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6940
6941	/* Restore the effects of TPR patching if any. */
6942	if (pVM->hm.s.fTPRPatchingActive)
6943	{
6944	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6945	AssertRC(rc);
6946	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6947	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6948	}
6949
6950	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6951	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6952
6953	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6954	uint32_t uExitReason;
6955	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6956	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6957	AssertRC(rc);
6958	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6959	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6960
6961	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6962	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6963
6964	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6965	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6966	{
6967	Log4(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6968	return;
6969	}
6970
6971	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6972	{
6973	/* Update the guest interruptibility-state from the VMCS. */
6974	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6975	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6976	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6977	AssertRC(rc);
6978	#endif
6979	/*
6980	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6981	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6982	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6983	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6984	*/
6985	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6986	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6987	{
6988	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6989	AssertRC(rc);
6990	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6991	}
6992	}
6993	}
6994
6995
6996	/**
6997	* Runs the guest code using VT-x.
6998	*
6999	* @returns VBox status code.
7000	* @param pVM Pointer to the VM.
7001	* @param pVCpu Pointer to the VMCPU.
7002	* @param pCtx Pointer to the guest-CPU context.
7003	*
7004	* @remarks Called with preemption disabled.
7005	*/
7006	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7007	{
7008	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7009	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7010
7011	VMXTRANSIENT VmxTransient;
7012	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7013	int rc = VERR_INTERNAL_ERROR_5;
7014	uint32_t cLoops = 0;
7015
7016	for (;; cLoops++)
7017	{
7018	Assert(!HMR0SuspendPending());
7019	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7020	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7021	(unsigned)RTMpCpuId(), cLoops));
7022
7023	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7024	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7025	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7026	if (rc != VINF_SUCCESS)
7027	break;
7028
7029	/*
7030	* No longjmps to ring-3 from this point on!!!
7031	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7032	* This also disables flushing of the R0-logger instance (if any).
7033	*/
7034	VMMRZCallRing3Disable(pVCpu);
7035	VMMRZCallRing3RemoveNotification(pVCpu);
7036	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7037
7038	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7039	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7040
7041	/*
7042	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7043	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7044	*/
7045	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7046	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7047	{
7048	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7049	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7050	return rc;
7051	}
7052
7053	/* Handle the VM-exit. */
7054	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7055	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7056	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7057	HMVMX_START_EXIT_DISPATCH_PROF();
7058	#ifdef HMVMX_USE_FUNCTION_TABLE
7059	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7060	#else
7061	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7062	#endif
7063	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7064	if (rc != VINF_SUCCESS)
7065	break;
7066	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7067	{
7068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7069	rc = VINF_EM_RAW_INTERRUPT;
7070	break;
7071	}
7072	}
7073
7074	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7075	if (rc == VERR_EM_INTERPRETER)
7076	rc = VINF_EM_RAW_EMULATE_INSTR;
7077	else if (rc == VINF_EM_RESET)
7078	rc = VINF_EM_TRIPLE_FAULT;
7079	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7080	return rc;
7081	}
7082
7083
7084	#ifndef HMVMX_USE_FUNCTION_TABLE
7085	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7086	{
7087	int rc;
7088	switch (rcReason)
7089	{
7090	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7091	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7092	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7093	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7094	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7095	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7096	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7097	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7098	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7099	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7100	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7101	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7102	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7103	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7104	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7105	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7106	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7107	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7108	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7109	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7110	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7111	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7112	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7113	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7114	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7115	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7116	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7117	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7118	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7119	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7120	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7121	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7122	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7123
7124	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7125	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7126	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7127	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7128	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7129	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7130	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7131	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7132	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7133
7134	case VMX_EXIT_VMCALL:
7135	case VMX_EXIT_VMCLEAR:
7136	case VMX_EXIT_VMLAUNCH:
7137	case VMX_EXIT_VMPTRLD:
7138	case VMX_EXIT_VMPTRST:
7139	case VMX_EXIT_VMREAD:
7140	case VMX_EXIT_VMRESUME:
7141	case VMX_EXIT_VMWRITE:
7142	case VMX_EXIT_VMXOFF:
7143	case VMX_EXIT_VMXON:
7144	case VMX_EXIT_INVEPT:
7145	case VMX_EXIT_INVVPID:
7146	case VMX_EXIT_VMFUNC:
7147	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7148	break;
7149	default:
7150	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7151	break;
7152	}
7153	return rc;
7154	}
7155	#endif
7156
7157	#ifdef DEBUG
7158	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7159	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7160	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7161	# define VMX_ASSERT_PREEMPT_CPUID() \
7162	do \
7163	{ \
7164	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7165	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7166	} while (0)
7167
7168	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7169	do { \
7170	AssertPtr(pVCpu); \
7171	AssertPtr(pMixedCtx); \
7172	AssertPtr(pVmxTransient); \
7173	Assert(pVmxTransient->fVMEntryFailed == false); \
7174	Assert(ASMIntAreEnabled()); \
7175	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7176	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7177	Log4Func(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7178	(unsigned)pVCpu->idCpu)); \
7179	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7180	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7181	VMX_ASSERT_PREEMPT_CPUID(); \
7182	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7183	} while (0)
7184	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7185	do { \
7186	Log4Func(("\n")); \
7187	} while(0)
7188	#else /* Release builds */
7189	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7190	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7191	#endif
7192
7193
7194	/**
7195	* Advances the guest RIP after reading it from the VMCS.
7196	*
7197	* @returns VBox status code.
7198	* @param pVCpu Pointer to the VMCPU.
7199	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7200	* out-of-sync. Make sure to update the required fields
7201	* before using them.
7202	* @param pVmxTransient Pointer to the VMX transient structure.
7203	*
7204	* @remarks No-long-jump zone!!!
7205	*/
7206	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7207	{
7208	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7209	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7210	AssertRCReturn(rc, rc);
7211
7212	pMixedCtx->rip += pVmxTransient->cbInstr;
7213	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7214	return rc;
7215	}
7216
7217
7218	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7219	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7220	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7221	/**
7222	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7223	*/
7224	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7225	{
7226	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7228	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7229	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7230	Assert(ASMIntAreEnabled());
7231	return VINF_SUCCESS;
7232	#else
7233	return VINF_EM_RAW_INTERRUPT;
7234	#endif
7235	}
7236
7237
7238	/**
7239	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7240	*/
7241	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7242	{
7243	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7244	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7245
7246	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7247	AssertRCReturn(rc, rc);
7248
7249	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7250	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7251	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7252
7253	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7254	{
7255	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7256	return VINF_EM_RAW_INTERRUPT;
7257	}
7258
7259	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7260	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7261	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7262	{
7263	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7264	return VINF_SUCCESS;
7265	}
7266	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7267	{
7268	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7269	return rc;
7270	}
7271
7272	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7273	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7274	switch (uIntrType)
7275	{
7276	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7277	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7278	/* no break */
7279	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7280	{
7281	switch (uVector)
7282	{
7283	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7284	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7285	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7286	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7287	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7288	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7289	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7290	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7291	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7292	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7293	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7294	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7295	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7296	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7297	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7298	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7299	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7300	#endif
7301	default:
7302	{
7303	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7304	AssertRCReturn(rc, rc);
7305
7306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7307	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7308	{
7309	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7310	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7311	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7312	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7313	AssertRCReturn(rc, rc);
7314	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7315	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7316	0 /* GCPtrFaultAddress */);
7317	AssertRCReturn(rc, rc);
7318	}
7319	else
7320	{
7321	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7322	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7323	}
7324	break;
7325	}
7326	}
7327	break;
7328	}
7329
7330	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7331	default:
7332	{
7333	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7334	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7335	break;
7336	}
7337	}
7338	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7339	return rc;
7340	}
7341
7342
7343	/**
7344	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7345	*/
7346	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7347	{
7348	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7349
7350	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7351	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7352	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7353	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7354	AssertRCReturn(rc, rc);
7355
7356	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7357	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7358	return VINF_SUCCESS;
7359	}
7360
7361
7362	/**
7363	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7364	*/
7365	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7366	{
7367	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7368	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7369	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7370	}
7371
7372
7373	/**
7374	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7375	*/
7376	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7377	{
7378	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7380	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7381	}
7382
7383
7384	/**
7385	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7386	*/
7387	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7388	{
7389	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7391	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7392	}
7393
7394
7395	/**
7396	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7397	*/
7398	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7399	{
7400	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7401	PVM pVM = pVCpu->CTX_SUFF(pVM);
7402	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7403	if (RT_LIKELY(rc == VINF_SUCCESS))
7404	{
7405	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7406	Assert(pVmxTransient->cbInstr == 2);
7407	}
7408	else
7409	{
7410	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7411	rc = VERR_EM_INTERPRETER;
7412	}
7413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7414	return rc;
7415	}
7416
7417
7418	/**
7419	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7420	*/
7421	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7422	{
7423	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7424	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7425	AssertRCReturn(rc, rc);
7426
7427	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7428	return VINF_EM_RAW_EMULATE_INSTR;
7429
7430	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7431	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7432	}
7433
7434
7435	/**
7436	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7437	*/
7438	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7439	{
7440	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7441	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7442	AssertRCReturn(rc, rc);
7443
7444	PVM pVM = pVCpu->CTX_SUFF(pVM);
7445	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7446	if (RT_LIKELY(rc == VINF_SUCCESS))
7447	{
7448	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7449	Assert(pVmxTransient->cbInstr == 2);
7450	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7451	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7452	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7453	}
7454	else
7455	{
7456	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7457	rc = VERR_EM_INTERPRETER;
7458	}
7459	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7460	return rc;
7461	}
7462
7463
7464	/**
7465	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7466	*/
7467	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7468	{
7469	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7470	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7471	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7472	AssertRCReturn(rc, rc);
7473
7474	PVM pVM = pVCpu->CTX_SUFF(pVM);
7475	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7476	if (RT_LIKELY(rc == VINF_SUCCESS))
7477	{
7478	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7479	Assert(pVmxTransient->cbInstr == 3);
7480	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7481	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7482	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7483	}
7484	else
7485	{
7486	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7487	rc = VERR_EM_INTERPRETER;
7488	}
7489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7490	return rc;
7491	}
7492
7493
7494	/**
7495	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7496	*/
7497	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7498	{
7499	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7500	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7501	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7502	AssertRCReturn(rc, rc);
7503
7504	PVM pVM = pVCpu->CTX_SUFF(pVM);
7505	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7506	if (RT_LIKELY(rc == VINF_SUCCESS))
7507	{
7508	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7509	Assert(pVmxTransient->cbInstr == 2);
7510	}
7511	else
7512	{
7513	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7514	rc = VERR_EM_INTERPRETER;
7515	}
7516	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7517	return rc;
7518	}
7519
7520
7521	/**
7522	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7523	*/
7524	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7525	{
7526	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7527	PVM pVM = pVCpu->CTX_SUFF(pVM);
7528	Assert(!pVM->hm.s.fNestedPaging);
7529
7530	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7531	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7532	AssertRCReturn(rc, rc);
7533
7534	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7535	rc = VBOXSTRICTRC_VAL(rc2);
7536	if (RT_LIKELY(rc == VINF_SUCCESS))
7537	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7538	else
7539	{
7540	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7541	pVmxTransient->uExitQualification, rc));
7542	}
7543	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7544	return rc;
7545	}
7546
7547
7548	/**
7549	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7550	*/
7551	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7552	{
7553	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7554	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7555	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7556	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7557	AssertRCReturn(rc, rc);
7558
7559	PVM pVM = pVCpu->CTX_SUFF(pVM);
7560	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7561	if (RT_LIKELY(rc == VINF_SUCCESS))
7562	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7563	else
7564	{
7565	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7566	rc = VERR_EM_INTERPRETER;
7567	}
7568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7569	return rc;
7570	}
7571
7572
7573	/**
7574	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7575	*/
7576	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7577	{
7578	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7579	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7580	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7581	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7582	AssertRCReturn(rc, rc);
7583
7584	PVM pVM = pVCpu->CTX_SUFF(pVM);
7585	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7586	rc = VBOXSTRICTRC_VAL(rc2);
7587	if (RT_LIKELY( rc == VINF_SUCCESS
7588	\|\| rc == VINF_EM_HALT))
7589	{
7590	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7591	AssertRCReturn(rc3, rc3);
7592
7593	if ( rc == VINF_EM_HALT
7594	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7595	{
7596	rc = VINF_SUCCESS;
7597	}
7598	}
7599	else
7600	{
7601	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7602	rc = VERR_EM_INTERPRETER;
7603	}
7604	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7605	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7607	return rc;
7608	}
7609
7610
7611	/**
7612	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7613	*/
7614	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7615	{
7616	/*
7617	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7618	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7619	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7620	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7621	*/
7622	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7623	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7624	}
7625
7626
7627	/**
7628	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7629	*/
7630	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7631	{
7632	/*
7633	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7634	* root operation. If we get there there is something funny going on.
7635	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7636	*/
7637	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7638	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7639	}
7640
7641
7642	/**
7643	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7644	*/
7645	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7646	{
7647	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7648	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7649	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7650	}
7651
7652
7653	/**
7654	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7655	*/
7656	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7657	{
7658	/*
7659	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7660	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7661	* See Intel spec. 25.3 "Other Causes of VM-exits".
7662	*/
7663	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7664	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7665	}
7666
7667
7668	/**
7669	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7670	* VM-exit.
7671	*/
7672	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7673	{
7674	/*
7675	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7676	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7677	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7678	*/
7679	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7680	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7681	}
7682
7683
7684	/**
7685	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7686	* VM-exit.
7687	*/
7688	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7689	{
7690	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7691	return VINF_EM_RESET;
7692	}
7693
7694
7695	/**
7696	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7697	*/
7698	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7699	{
7700	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7701	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7702	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7703	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7704	AssertRCReturn(rc, rc);
7705
7706	pMixedCtx->rip++;
7707	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7708	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7709	rc = VINF_SUCCESS;
7710	else
7711	rc = VINF_EM_HALT;
7712
7713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7714	return rc;
7715	}
7716
7717
7718	/**
7719	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7720	*/
7721	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7722	{
7723	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7724	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7725	return VINF_SUCCESS;
7726	}
7727
7728
7729	/**
7730	* VM-exit handler for expiry of the VMX preemption timer.
7731	*/
7732	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7733	{
7734	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7735
7736	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7737	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7738
7739	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7740	PVM pVM = pVCpu->CTX_SUFF(pVM);
7741	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7743	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7744	}
7745
7746
7747	/**
7748	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7749	*/
7750	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7751	{
7752	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7753
7754	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7755	/** @todo check if XSETBV is supported by the recompiler. */
7756	return VERR_EM_INTERPRETER;
7757	}
7758
7759
7760	/**
7761	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7762	*/
7763	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7764	{
7765	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7766
7767	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7768	/** @todo implement EMInterpretInvpcid() */
7769	return VERR_EM_INTERPRETER;
7770	}
7771
7772
7773	/**
7774	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7775	* Error VM-exit.
7776	*/
7777	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7778	{
7779	uint32_t uIntrState;
7780	HMVMXHCUINTREG uHCReg;
7781	uint64_t u64Val;
7782	uint32_t u32Val;
7783
7784	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7785	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7786	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7787	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7788	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7789	AssertRCReturn(rc, rc);
7790
7791	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7792	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7793	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7794	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7795
7796	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7797	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7798	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7799	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7800	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7801	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7802	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7803	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7804	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7805	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7806	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7807	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7808
7809	PVM pVM = pVCpu->CTX_SUFF(pVM);
7810	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7811
7812	return VERR_VMX_INVALID_GUEST_STATE;
7813	}
7814
7815
7816	/**
7817	* VM-exit handler for VM-entry failure due to an MSR-load
7818	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7819	*/
7820	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7821	{
7822	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7823	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7824	}
7825
7826
7827	/**
7828	* VM-exit handler for VM-entry failure due to a machine-check event
7829	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7830	*/
7831	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7832	{
7833	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7834	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7835	}
7836
7837
7838	/**
7839	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7840	* theory.
7841	*/
7842	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7843	{
7844	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7845	return VERR_VMX_UNDEFINED_EXIT_CODE;
7846	}
7847
7848
7849	/**
7850	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7851	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7852	* Conditional VM-exit.
7853	*/
7854	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7855	{
7856	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7857
7858	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7859	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7860	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7861	return VERR_EM_INTERPRETER;
7862	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7863	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7864	}
7865
7866
7867	/**
7868	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7869	*/
7870	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7871	{
7872	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7873
7874	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7875	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7876	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7877	return VERR_EM_INTERPRETER;
7878	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7879	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7880	}
7881
7882
7883	/**
7884	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7885	*/
7886	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7887	{
7888	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7889
7890	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7891	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7892	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7893	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7894	AssertRCReturn(rc, rc);
7895
7896	PVM pVM = pVCpu->CTX_SUFF(pVM);
7897	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7898	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7899	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7901
7902	if (RT_LIKELY(rc == VINF_SUCCESS))
7903	{
7904	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7905	Assert(pVmxTransient->cbInstr == 2);
7906	}
7907	return rc;
7908	}
7909
7910
7911	/**
7912	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7913	*/
7914	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7915	{
7916	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7917	PVM pVM = pVCpu->CTX_SUFF(pVM);
7918	int rc = VINF_SUCCESS;
7919
7920	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7921	if ( pVM->hm.s.fTPRPatchingActive
7922	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7923	{
7924	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7925	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7926	{
7927	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7928	AssertRC(rc);
7929	}
7930
7931	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7932	Assert(pVmxTransient->cbInstr == 2);
7933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7934	return VINF_SUCCESS;
7935	}
7936
7937	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7938	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7939	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7940	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7941	AssertRCReturn(rc, rc);
7942	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
7943
7944	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7945	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7947
7948	if (RT_LIKELY(rc == VINF_SUCCESS))
7949	{
7950	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7951
7952	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7953	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7954	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7955	{
7956	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7957	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7958	}
7959	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7960	{
7961	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7962	AssertRCReturn(rc, rc);
7963	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7964	}
7965	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7966	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7967
7968	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7969	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7970	{
7971	switch (pMixedCtx->ecx)
7972	{
7973	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7974	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7975	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7976	case MSR_K8_FS_BASE: /* no break */
7977	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7978	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7979	}
7980	}
7981	#ifdef VBOX_STRICT
7982	else
7983	{
7984	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7985	switch (pMixedCtx->ecx)
7986	{
7987	case MSR_IA32_SYSENTER_CS:
7988	case MSR_IA32_SYSENTER_EIP:
7989	case MSR_IA32_SYSENTER_ESP:
7990	case MSR_K8_FS_BASE:
7991	case MSR_K8_GS_BASE:
7992	{
7993	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7994	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7995	}
7996
7997	case MSR_K8_LSTAR:
7998	case MSR_K6_STAR:
7999	case MSR_K8_SF_MASK:
8000	case MSR_K8_TSC_AUX:
8001	case MSR_K8_KERNEL_GS_BASE:
8002	{
8003	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8004	pMixedCtx->ecx));
8005	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8006	}
8007	}
8008	}
8009	#endif /* VBOX_STRICT */
8010	}
8011	return rc;
8012	}
8013
8014
8015	/**
8016	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8017	*/
8018	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8019	{
8020	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8021
8022	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8023	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8024	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8025	return VERR_EM_INTERPRETER;
8026	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8027	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8028	}
8029
8030
8031	/**
8032	* VM-exit handler for when the TPR value is lowered below the specified
8033	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8034	*/
8035	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8036	{
8037	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8038	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8039
8040	/*
8041	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8042	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8043	* resume guest execution.
8044	*/
8045	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8046	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8047	return VINF_SUCCESS;
8048	}
8049
8050
8051	/**
8052	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8053	* VM-exit.
8054	*
8055	* @retval VINF_SUCCESS when guest execution can continue.
8056	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8057	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8058	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8059	* recompiler.
8060	*/
8061	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8062	{
8063	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8064	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8065	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8066	AssertRCReturn(rc, rc);
8067
8068	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8069	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8070	PVM pVM = pVCpu->CTX_SUFF(pVM);
8071	switch (uAccessType)
8072	{
8073	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8074	{
8075	#if 0
8076	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8077	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8078	#else
8079	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8080	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8081	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8082	#endif
8083	AssertRCReturn(rc, rc);
8084
8085	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8086	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8087	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8088	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8089
8090	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8091	{
8092	case 0: /* CR0 */
8093	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8094	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8095	break;
8096	case 2: /* C2 **/
8097	/* Nothing to do here, CR2 it's not part of the VMCS. */
8098	break;
8099	case 3: /* CR3 */
8100	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8101	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8102	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8103	break;
8104	case 4: /* CR4 */
8105	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8106	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8107	break;
8108	case 8: /* CR8 */
8109	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8110	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8111	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8112	break;
8113	default:
8114	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8115	break;
8116	}
8117
8118	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8119	break;
8120	}
8121
8122	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8123	{
8124	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8125	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8126	AssertRCReturn(rc, rc);
8127	Assert( !pVM->hm.s.fNestedPaging
8128	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8129	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8130
8131	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8132	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8133	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8134
8135	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8136	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8137	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8138	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8139	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8140	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8141	break;
8142	}
8143
8144	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8145	{
8146	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8147	AssertRCReturn(rc, rc);
8148	rc = EMInterpretCLTS(pVM, pVCpu);
8149	AssertRCReturn(rc, rc);
8150	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8151	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8152	Log4(("CRX CLTS write rc=%d\n", rc));
8153	break;
8154	}
8155
8156	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8157	{
8158	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8159	AssertRCReturn(rc, rc);
8160	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8161	if (RT_LIKELY(rc == VINF_SUCCESS))
8162	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8163	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8164	Log4(("CRX LMSW write rc=%d\n", rc));
8165	break;
8166	}
8167
8168	default:
8169	{
8170	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8171	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8172	}
8173	}
8174
8175	/* Validate possible error codes. */
8176	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8177	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8178	if (RT_SUCCESS(rc))
8179	{
8180	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8181	AssertRCReturn(rc2, rc2);
8182	}
8183
8184	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8185	return rc;
8186	}
8187
8188
8189	/**
8190	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8191	* VM-exit.
8192	*/
8193	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8194	{
8195	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8196	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8197
8198	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8199	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8200	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8201	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8202	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8203	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8204	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8205	AssertRCReturn(rc, rc);
8206
8207	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8208
8209	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8210	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8211	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8212	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8213	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8214	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8215	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8216
8217	/* I/O operation lookup arrays. */
8218	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8219	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8220
8221	const uint32_t cbSize = s_aIOSize[uIOWidth];
8222	const uint32_t cbInstr = pVmxTransient->cbInstr;
8223	PVM pVM = pVCpu->CTX_SUFF(pVM);
8224	if (fIOString)
8225	{
8226	/* INS/OUTS - I/O String instruction. */
8227	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8228	/** @todo for now manually disassemble later optimize by getting the fields from
8229	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8230	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8231	* segment prefix info. */
8232	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8233	if (RT_SUCCESS(rc))
8234	{
8235	if (fIOWrite)
8236	{
8237	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8238	(DISCPUMODE)pDis->uAddrMode, cbSize);
8239	rc = VBOXSTRICTRC_VAL(rc2);
8240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8241	}
8242	else
8243	{
8244	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8245	(DISCPUMODE)pDis->uAddrMode, cbSize);
8246	rc = VBOXSTRICTRC_VAL(rc2);
8247	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8248	}
8249	}
8250	else
8251	{
8252	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8253	rc = VINF_EM_RAW_EMULATE_INSTR;
8254	}
8255	}
8256	else
8257	{
8258	/* IN/OUT - I/O instruction. */
8259	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8260	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8261	if (fIOWrite)
8262	{
8263	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8264	rc = VBOXSTRICTRC_VAL(rc2);
8265	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8266	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8268	}
8269	else
8270	{
8271	uint32_t u32Result = 0;
8272	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8273	rc = VBOXSTRICTRC_VAL(rc2);
8274	if (IOM_SUCCESS(rc))
8275	{
8276	/* Save result of I/O IN instr. in AL/AX/EAX. */
8277	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8278	}
8279	else if (rc == VINF_IOM_R3_IOPORT_READ)
8280	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8281	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8282	}
8283	}
8284
8285	if (IOM_SUCCESS(rc))
8286	{
8287	pMixedCtx->rip += cbInstr;
8288	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8289	if (RT_LIKELY(rc == VINF_SUCCESS))
8290	{
8291	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8292	AssertRCReturn(rc, rc);
8293
8294	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8295	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8296	{
8297	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8298	for (unsigned i = 0; i < 4; i++)
8299	{
8300	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8301	if ( ( uIOPort >= pMixedCtx->dr[i]
8302	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8303	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8304	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8305	{
8306	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8307	uint64_t uDR6 = ASMGetDR6();
8308
8309	/* Clear all breakpoint status flags and set the one we just hit. */
8310	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8311	uDR6 \|= (uint64_t)RT_BIT(i);
8312
8313	/*
8314	* Note: AMD64 Architecture Programmer's Manual 13.1:
8315	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8316	* be cleared by software after the contents have been read.
8317	*/
8318	ASMSetDR6(uDR6);
8319
8320	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8321	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8322
8323	/* Paranoia. */
8324	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8325	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8326	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8327
8328	/* Resync DR7 */
8329	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8330	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8331
8332	/* Set #DB to be injected into the VM and continue guest execution. */
8333	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8334	break;
8335	}
8336	}
8337	}
8338	}
8339	}
8340
8341	#ifdef DEBUG
8342	if (rc == VINF_IOM_R3_IOPORT_READ)
8343	Assert(!fIOWrite);
8344	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8345	Assert(fIOWrite);
8346	else
8347	{
8348	AssertMsg( RT_FAILURE(rc)
8349	\|\| rc == VINF_SUCCESS
8350	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8351	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8352	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8353	}
8354	#endif
8355
8356	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8357	return rc;
8358	}
8359
8360
8361	/**
8362	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8363	* VM-exit.
8364	*/
8365	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8366	{
8367	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8368
8369	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8370	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8371	AssertRCReturn(rc, rc);
8372	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8373	{
8374	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8375	AssertRCReturn(rc, rc);
8376	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8377	{
8378	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8379
8380	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8381	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8382	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8383	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8384	{
8385	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8386	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8387
8388	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8389	Assert(!pVCpu->hm.s.Event.fPending);
8390	pVCpu->hm.s.Event.fPending = true;
8391	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8392	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8393	AssertRCReturn(rc, rc);
8394	if (fErrorCodeValid)
8395	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8396	else
8397	pVCpu->hm.s.Event.u32ErrCode = 0;
8398	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8399	&& uVector == X86_XCPT_PF)
8400	{
8401	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8402	}
8403
8404	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8405	}
8406	}
8407	}
8408
8409	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8410	* emulation. */
8411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8412	return VERR_EM_INTERPRETER;
8413	}
8414
8415
8416	/**
8417	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8418	*/
8419	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8420	{
8421	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8422	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8423	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8424	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8425	AssertRCReturn(rc, rc);
8426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8427	return VINF_EM_DBG_STOP;
8428	}
8429
8430
8431	/**
8432	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8433	*/
8434	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8435	{
8436	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8437
8438	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8439	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8440	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8441	return VINF_SUCCESS;
8442	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8443	return rc;
8444
8445	#if 0
8446	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8447	* just sync the whole thing. */
8448	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8449	#else
8450	/* Aggressive state sync. for now. */
8451	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8452	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8453	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8454	#endif
8455	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8456	AssertRCReturn(rc, rc);
8457
8458	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8459	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8460	switch (uAccessType)
8461	{
8462	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8463	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8464	{
8465	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8466	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8467	{
8468	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8469	}
8470
8471	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8472	GCPhys &= PAGE_BASE_GC_MASK;
8473	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8474	PVM pVM = pVCpu->CTX_SUFF(pVM);
8475	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8476	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8477
8478	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8479	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8480	CPUMCTX2CORE(pMixedCtx), GCPhys);
8481	rc = VBOXSTRICTRC_VAL(rc2);
8482	Log4(("ApicAccess rc=%d\n", rc));
8483	if ( rc == VINF_SUCCESS
8484	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8485	\|\| rc == VERR_PAGE_NOT_PRESENT)
8486	{
8487	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8488	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8489	rc = VINF_SUCCESS;
8490	}
8491	break;
8492	}
8493
8494	default:
8495	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8496	rc = VINF_EM_RAW_EMULATE_INSTR;
8497	break;
8498	}
8499
8500	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8501	return rc;
8502	}
8503
8504
8505	/**
8506	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8507	* VM-exit.
8508	*/
8509	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8510	{
8511	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8512
8513	/* We should -not- get this VM-exit if the guest is debugging. */
8514	if (CPUMIsGuestDebugStateActive(pVCpu))
8515	{
8516	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8517	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8518	}
8519
8520	int rc = VERR_INTERNAL_ERROR_5;
8521	if ( !DBGFIsStepping(pVCpu)
8522	&& !CPUMIsHyperDebugStateActive(pVCpu))
8523	{
8524	/* Don't intercept MOV DRx. */
8525	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8526	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8527	AssertRCReturn(rc, rc);
8528
8529	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8530	PVM pVM = pVCpu->CTX_SUFF(pVM);
8531	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8532	AssertRC(rc);
8533	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8534
8535	#ifdef VBOX_WITH_STATISTICS
8536	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8537	AssertRCReturn(rc, rc);
8538	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8540	else
8541	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8542	#endif
8543	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8544	return VINF_SUCCESS;
8545	}
8546
8547	/*
8548	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8549	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8550	*/
8551	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8552	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8553	AssertRCReturn(rc, rc);
8554
8555	PVM pVM = pVCpu->CTX_SUFF(pVM);
8556	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8557	{
8558	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8559	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8560	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8561	if (RT_SUCCESS(rc))
8562	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8564	}
8565	else
8566	{
8567	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8568	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8569	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8571	}
8572
8573	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8574	if (RT_SUCCESS(rc))
8575	{
8576	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8577	AssertRCReturn(rc2, rc2);
8578	}
8579	return rc;
8580	}
8581
8582
8583	/**
8584	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8585	* Conditional VM-exit.
8586	*/
8587	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8588	{
8589	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8590	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8591
8592	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8593	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8594	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8595	return VINF_SUCCESS;
8596	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8597	return rc;
8598
8599	RTGCPHYS GCPhys = 0;
8600	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8601
8602	#if 0
8603	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8604	#else
8605	/* Aggressive state sync. for now. */
8606	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8607	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8608	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8609	#endif
8610	AssertRCReturn(rc, rc);
8611
8612	/*
8613	* If we succeed, resume guest execution.
8614	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8615	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8616	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8617	* weird case. See @bugref{6043}.
8618	*/
8619	PVM pVM = pVCpu->CTX_SUFF(pVM);
8620	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8621	rc = VBOXSTRICTRC_VAL(rc2);
8622	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8623	if ( rc == VINF_SUCCESS
8624	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8625	\|\| rc == VERR_PAGE_NOT_PRESENT)
8626	{
8627	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8628	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8629	return VINF_SUCCESS;
8630	}
8631	return rc;
8632	}
8633
8634
8635	/**
8636	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8637	* VM-exit.
8638	*/
8639	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8640	{
8641	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8642	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8643
8644	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8645	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8646	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8647	return VINF_SUCCESS;
8648	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8649	return rc;
8650
8651	RTGCPHYS GCPhys = 0;
8652	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8653	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8654	#if 0
8655	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8656	#else
8657	/* Aggressive state sync. for now. */
8658	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8659	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8660	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8661	#endif
8662	AssertRCReturn(rc, rc);
8663
8664	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8665	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8666
8667	RTGCUINT uErrorCode = 0;
8668	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8669	uErrorCode \|= X86_TRAP_PF_ID;
8670	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8671	uErrorCode \|= X86_TRAP_PF_RW;
8672	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8673	uErrorCode \|= X86_TRAP_PF_P;
8674
8675	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8676
8677	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8678	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8679
8680	/* Handle the pagefault trap for the nested shadow table. */
8681	PVM pVM = pVCpu->CTX_SUFF(pVM);
8682	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8683	TRPMResetTrap(pVCpu);
8684
8685	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8686	if ( rc == VINF_SUCCESS
8687	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8688	\|\| rc == VERR_PAGE_NOT_PRESENT)
8689	{
8690	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8691	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8692	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8693	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8694	return VINF_SUCCESS;
8695	}
8696
8697	Log4(("EPT return to ring-3 rc=%d\n"));
8698	return rc;
8699	}
8700
8701
8702	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8703	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8704	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8705	/**
8706	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8707	*/
8708	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8709	{
8710	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8712
8713	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8714	AssertRCReturn(rc, rc);
8715
8716	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8717	{
8718	/* Old-style FPU error reporting needs some extra work. */
8719	/** @todo don't fall back to the recompiler, but do it manually. */
8720	return VERR_EM_INTERPRETER;
8721	}
8722	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8723	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8724	return rc;
8725	}
8726
8727
8728	/**
8729	* VM-exit exception handler for #BP (Breakpoint exception).
8730	*/
8731	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8732	{
8733	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8734	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8735
8736	/** @todo Try optimize this by not saving the entire guest state unless
8737	* really needed. */
8738	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8739	AssertRCReturn(rc, rc);
8740
8741	PVM pVM = pVCpu->CTX_SUFF(pVM);
8742	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8743	if (rc == VINF_EM_RAW_GUEST_TRAP)
8744	{
8745	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8746	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8747	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8748	AssertRCReturn(rc, rc);
8749
8750	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8751	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8752	}
8753
8754	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8755	return rc;
8756	}
8757
8758
8759	/**
8760	* VM-exit exception handler for #DB (Debug exception).
8761	*/
8762	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8763	{
8764	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8765	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8766
8767	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8768	AssertRCReturn(rc, rc);
8769
8770	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8771	uint64_t uDR6 = X86_DR6_INIT_VAL;
8772	uDR6 \|= (pVmxTransient->uExitQualification
8773	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8774	PVM pVM = pVCpu->CTX_SUFF(pVM);
8775	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8776	if (rc == VINF_EM_RAW_GUEST_TRAP)
8777	{
8778	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8779	pMixedCtx->dr[6] = uDR6;
8780
8781	if (CPUMIsGuestDebugStateActive(pVCpu))
8782	ASMSetDR6(pMixedCtx->dr[6]);
8783
8784	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8785
8786	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8787	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8788
8789	/* Paranoia. */
8790	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8791	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8792	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8793
8794	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8795	AssertRCReturn(rc,rc);
8796
8797	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8798	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8799	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8800	AssertRCReturn(rc2, rc2);
8801	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8802	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8803	rc = VINF_SUCCESS;
8804	}
8805
8806	return rc;
8807	}
8808
8809
8810	/**
8811	* VM-exit exception handler for #NM (Device-not-available exception: floating
8812	* point exception).
8813	*/
8814	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8815	{
8816	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8817
8818	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8819	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8820	#endif
8821
8822	/* We require CR0 and EFER. EFER is always up-to-date. */
8823	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8824	AssertRCReturn(rc, rc);
8825
8826	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8827	PVM pVM = pVCpu->CTX_SUFF(pVM);
8828	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8829	if (rc == VINF_SUCCESS)
8830	{
8831	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8832	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8834	return VINF_SUCCESS;
8835	}
8836
8837	/* Forward #NM to the guest. */
8838	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8839	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8840	AssertRCReturn(rc, rc);
8841	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8842	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8843	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8844	return rc;
8845	}
8846
8847
8848	/**
8849	* VM-exit exception handler for #GP (General-protection exception).
8850	*
8851	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8852	*/
8853	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8854	{
8855	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8857
8858	int rc = VERR_INTERNAL_ERROR_5;
8859	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8860	{
8861	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8862	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8863	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8864	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8865	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8866	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8867	AssertRCReturn(rc, rc);
8868	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8869	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8870	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8871	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8872	return rc;
8873	#else
8874	/* We don't intercept #GP. */
8875	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8876	return VERR_VMX_UNEXPECTED_EXCEPTION;
8877	#endif
8878	}
8879
8880	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8881	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8882
8883	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8884	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8885	AssertRCReturn(rc, rc);
8886
8887	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8888	uint32_t cbOp = 0;
8889	PVM pVM = pVCpu->CTX_SUFF(pVM);
8890	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8891	if (RT_SUCCESS(rc))
8892	{
8893	rc = VINF_SUCCESS;
8894	Assert(cbOp == pDis->cbInstr);
8895	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8896	switch (pDis->pCurInstr->uOpcode)
8897	{
8898	case OP_CLI:
8899	{
8900	pMixedCtx->eflags.Bits.u1IF = 0;
8901	pMixedCtx->rip += pDis->cbInstr;
8902	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8904	break;
8905	}
8906
8907	case OP_STI:
8908	{
8909	pMixedCtx->eflags.Bits.u1IF = 1;
8910	pMixedCtx->rip += pDis->cbInstr;
8911	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8912	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8913	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8915	break;
8916	}
8917
8918	case OP_HLT:
8919	{
8920	rc = VINF_EM_HALT;
8921	pMixedCtx->rip += pDis->cbInstr;
8922	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8924	break;
8925	}
8926
8927	case OP_POPF:
8928	{
8929	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8930	uint32_t cbParm = 0;
8931	uint32_t uMask = 0;
8932	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8933	{
8934	cbParm = 4;
8935	uMask = 0xffffffff;
8936	}
8937	else
8938	{
8939	cbParm = 2;
8940	uMask = 0xffff;
8941	}
8942
8943	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8944	RTGCPTR GCPtrStack = 0;
8945	X86EFLAGS uEflags;
8946	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8947	&GCPtrStack);
8948	if (RT_SUCCESS(rc))
8949	{
8950	Assert(sizeof(uEflags.u32) >= cbParm);
8951	uEflags.u32 = 0;
8952	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8953	}
8954	if (RT_FAILURE(rc))
8955	{
8956	rc = VERR_EM_INTERPRETER;
8957	break;
8958	}
8959	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8960	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8961	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8962	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8963	pMixedCtx->eflags.Bits.u1RF = 0;
8964	pMixedCtx->esp += cbParm;
8965	pMixedCtx->esp &= uMask;
8966	pMixedCtx->rip += pDis->cbInstr;
8967	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8969	break;
8970	}
8971
8972	case OP_PUSHF:
8973	{
8974	uint32_t cbParm = 0;
8975	uint32_t uMask = 0;
8976	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8977	{
8978	cbParm = 4;
8979	uMask = 0xffffffff;
8980	}
8981	else
8982	{
8983	cbParm = 2;
8984	uMask = 0xffff;
8985	}
8986
8987	/* Get the stack pointer & push the contents of eflags onto the stack. */
8988	RTGCPTR GCPtrStack = 0;
8989	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8990	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8991	if (RT_FAILURE(rc))
8992	{
8993	rc = VERR_EM_INTERPRETER;
8994	break;
8995	}
8996	X86EFLAGS uEflags;
8997	uEflags = pMixedCtx->eflags;
8998	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8999	uEflags.Bits.u1RF = 0;
9000	uEflags.Bits.u1VM = 0;
9001
9002	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
9003	if (RT_FAILURE(rc))
9004	{
9005	rc = VERR_EM_INTERPRETER;
9006	break;
9007	}
9008	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9009	pMixedCtx->esp -= cbParm;
9010	pMixedCtx->esp &= uMask;
9011	pMixedCtx->rip += pDis->cbInstr;
9012	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9013	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9014	break;
9015	}
9016
9017	case OP_IRET:
9018	{
9019	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9020	* instruction reference. */
9021	RTGCPTR GCPtrStack = 0;
9022	uint32_t uMask = 0xffff;
9023	uint16_t aIretFrame[3];
9024	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9025	{
9026	rc = VERR_EM_INTERPRETER;
9027	break;
9028	}
9029	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9030	&GCPtrStack);
9031	if (RT_SUCCESS(rc))
9032	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9033	if (RT_FAILURE(rc))
9034	{
9035	rc = VERR_EM_INTERPRETER;
9036	break;
9037	}
9038	pMixedCtx->eip = 0;
9039	pMixedCtx->ip = aIretFrame[0];
9040	pMixedCtx->cs.Sel = aIretFrame[1];
9041	pMixedCtx->cs.ValidSel = aIretFrame[1];
9042	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9043	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9044	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9045	pMixedCtx->sp += sizeof(aIretFrame);
9046	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9047	\| HM_CHANGED_GUEST_RFLAGS;
9048	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9050	break;
9051	}
9052
9053	case OP_INT:
9054	{
9055	uint16_t uVector = pDis->Param1.uValue & 0xff;
9056	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9058	break;
9059	}
9060
9061	case OP_INTO:
9062	{
9063	if (pMixedCtx->eflags.Bits.u1OF)
9064	{
9065	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9066	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9067	}
9068	break;
9069	}
9070
9071	default:
9072	{
9073	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9074	EMCODETYPE_SUPERVISOR);
9075	rc = VBOXSTRICTRC_VAL(rc2);
9076	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9077	Log4(("#GP rc=%Rrc\n", rc));
9078	break;
9079	}
9080	}
9081	}
9082	else
9083	rc = VERR_EM_INTERPRETER;
9084
9085	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9086	("#GP Unexpected rc=%Rrc\n", rc));
9087	return rc;
9088	}
9089
9090
9091	/**
9092	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9093	* the exception reported in the VMX transient structure back into the VM.
9094	*
9095	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9096	* up-to-date.
9097	*/
9098	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9099	{
9100	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9101
9102	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9103	hmR0VmxCheckExitDueToEventDelivery(). */
9104	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9105	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9106	AssertRCReturn(rc, rc);
9107	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9108	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9109	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9110	return VINF_SUCCESS;
9111	}
9112
9113
9114	/**
9115	* VM-exit exception handler for #PF (Page-fault exception).
9116	*/
9117	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9118	{
9119	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9120	PVM pVM = pVCpu->CTX_SUFF(pVM);
9121	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9122	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9123	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9124	AssertRCReturn(rc, rc);
9125
9126	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9127	if (pVM->hm.s.fNestedPaging)
9128	{
9129	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9130	{
9131	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9132	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9133	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9134	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9135	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9136	}
9137	else
9138	{
9139	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9140	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
9141	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9142	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9143	}
9144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9145	return rc;
9146	}
9147	#else
9148	Assert(!pVM->hm.s.fNestedPaging);
9149	#endif
9150
9151	#ifdef VBOX_HM_WITH_GUEST_PATCHING
9152	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9153	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9154	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9155	AssertRCReturn(rc, rc);
9156	/* Shortcut for APIC TPR access, only for 32-bit guests. */
9157	if ( pVM->hm.s.fTRPPatchingAllowed
9158	&& pVM->hm.s.pGuestPatchMem
9159	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
9160	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
9161	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
9162	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
9163	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
9164	{
9165	RTGCPHYS GCPhys;
9166	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
9167	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9168	if ( rc == VINF_SUCCESS
9169	&& GCPhys == GCPhysApicBase)
9170	{
9171	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9172	AssertRCReturn(rc, rc);
9173
9174	/* Only attempt to patch the instruction once. */
9175	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9176	if (!pPatch)
9177	return VINF_EM_HM_PATCH_TPR_INSTR;
9178	}
9179	}
9180	#endif
9181
9182	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9183	AssertRCReturn(rc, rc);
9184
9185	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9186	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9187
9188	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9189	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9190	(RTGCPTR)pVmxTransient->uExitQualification);
9191
9192	Log4(("#PF: rc=%Rrc\n", rc));
9193	if (rc == VINF_SUCCESS)
9194	{
9195	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9196	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9197	* memory? We don't update the whole state here... */
9198	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9199	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9200	TRPMResetTrap(pVCpu);
9201	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9202	return rc;
9203	}
9204	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9205	{
9206	if (!pVmxTransient->fVectoringPF)
9207	{
9208	/* It's a guest page fault and needs to be reflected to the guest. */
9209	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9210	TRPMResetTrap(pVCpu);
9211	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9212	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9213	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9214	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9215	}
9216	else
9217	{
9218	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9219	TRPMResetTrap(pVCpu);
9220	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9221	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9222	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9223	}
9224
9225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9226	return VINF_SUCCESS;
9227	}
9228
9229	TRPMResetTrap(pVCpu);
9230	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9231	return rc;
9232	}
9233

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46502

Download in other formats: