HMVMXR0.cpp@ 46393

Last change on this file since 46393 was 46393, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Log4Func to keep things at level 4 and not flow or level 1 logging.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 385.5 KB

Line
1	/* $Id: HMVMXR0.cpp 46393 2013-06-05 11:20:24Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/**
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG uEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
174	uint64_t u64LStarMsr;
175	/** The guest's TPR value used for TPR shadowing. */
176	uint8_t u8GuestTpr;
177	/** Alignment. */
178	uint8_t abAlignment0[6];
179
180	/** The basic VM-exit reason. */
181	uint16_t uExitReason;
182	/** Alignment. */
183	uint16_t u16Alignment0;
184	/** The VM-exit interruption error code. */
185	uint32_t uExitIntrErrorCode;
186	/** The VM-exit exit qualification. */
187	uint64_t uExitQualification;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntrInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** Whether the VM-entry failed or not. */
194	bool fVMEntryFailed;
195	/** Alignment. */
196	uint8_t abAlignment1[5];
197
198	/** The VM-entry interruption-information field. */
199	uint32_t uEntryIntrInfo;
200	/** The VM-entry exception error code field. */
201	uint32_t uEntryXcptErrorCode;
202	/** The VM-entry instruction length field. */
203	uint32_t cbEntryInstr;
204
205	/** IDT-vectoring information field. */
206	uint32_t uIdtVectoringInfo;
207	/** IDT-vectoring error code. */
208	uint32_t uIdtVectoringErrorCode;
209
210	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
211	uint32_t fVmcsFieldsRead;
212	/** Whether TSC-offsetting should be setup before VM-entry. */
213	bool fUpdateTscOffsettingAndPreemptTimer;
214	/** Whether the VM-exit was caused by a page-fault during delivery of a
215	* contributary exception or a page-fault. */
216	bool fVectoringPF;
217	} VMXTRANSIENT, *PVMXTRANSIENT;
218	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
219	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
220	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
221
222
223	/**
224	* MSR-bitmap read permissions.
225	*/
226	typedef enum VMXMSREXITREAD
227	{
228	/** Reading this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_READ = 0xb,
230	/** Reading this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_READ
232	} VMXMSREXITREAD;
233
234	/**
235	* MSR-bitmap write permissions.
236	*/
237	typedef enum VMXMSREXITWRITE
238	{
239	/** Writing to this MSR causes a VM-exit. */
240	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
241	/** Writing to this MSR does not cause a VM-exit. */
242	VMXMSREXIT_PASSTHRU_WRITE
243	} VMXMSREXITWRITE;
244
245
246	/*******************************************************************************
247	* Internal Functions *
248	*******************************************************************************/
249	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
250	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
251	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
252	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
253	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
254	#endif
255	#ifndef HMVMX_USE_FUNCTION_TABLE
256	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
257	#define HMVMX_EXIT_DECL static int
258	#else
259	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
260	#endif
261
262	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306
307	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314
315
316	/*******************************************************************************
317	* Global Variables *
318	*******************************************************************************/
319	#ifdef HMVMX_USE_FUNCTION_TABLE
320	/**
321	* VM-exit handler.
322	*
323	* @returns VBox status code.
324	* @param pVCpu Pointer to the VMCPU.
325	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
326	* out-of-sync. Make sure to update the required
327	* fields before using them.
328	* @param pVmxTransient Pointer to the VMX-transient structure.
329	*/
330	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
333
334	/**
335	* VMX_EXIT dispatch table.
336	*/
337	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
338	{
339	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
340	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
341	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
342	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
343	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
344	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
345	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
346	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
347	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
348	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
349	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
350	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
351	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
352	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
353	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
354	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
355	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
356	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
357	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
358	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
359	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
360	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
361	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
362	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
363	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
364	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
365	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
366	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
367	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
368	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
369	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
370	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
371	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
372	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
373	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
374	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
376	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
377	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
378	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
379	/* 40 UNDEFINED */ hmR0VmxExitPause,
380	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
381	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
382	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
383	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
384	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
385	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
386	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
387	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
388	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
389	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
390	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
391	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
392	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
393	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
394	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
395	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
396	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
397	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
398	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
399	};
400	#endif /* HMVMX_USE_FUNCTION_TABLE */
401
402	#ifdef VBOX_STRICT
403	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
404	{
405	/* 0 */ "(Not Used)",
406	/* 1 */ "VMCALL executed in VMX root operation.",
407	/* 2 */ "VMCLEAR with invalid physical address.",
408	/* 3 */ "VMCLEAR with VMXON pointer.",
409	/* 4 */ "VMLAUNCH with non-clear VMCS.",
410	/* 5 */ "VMRESUME with non-launched VMCS.",
411	/* 6 */ "VMRESUME after VMXOFF",
412	/* 7 */ "VM entry with invalid control fields.",
413	/* 8 */ "VM entry with invalid host state fields.",
414	/* 9 */ "VMPTRLD with invalid physical address.",
415	/* 10 */ "VMPTRLD with VMXON pointer.",
416	/* 11 */ "VMPTRLD with incorrect revision identifier.",
417	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
418	/* 13 */ "VMWRITE to read-only VMCS component.",
419	/* 14 */ "(Not Used)",
420	/* 15 */ "VMXON executed in VMX root operation.",
421	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
422	/* 17 */ "VM entry with non-launched executing VMCS.",
423	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
424	/* 19 */ "VMCALL with non-clear VMCS.",
425	/* 20 */ "VMCALL with invalid VM-exit control fields.",
426	/* 21 */ "(Not Used)",
427	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
428	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
429	/* 24 */ "VMCALL with invalid SMM-monitor features.",
430	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
431	/* 26 */ "VM entry with events blocked by MOV SS.",
432	/* 27 */ "(Not Used)",
433	/* 28 */ "Invalid operand to INVEPT/INVVPID."
434	};
435	#endif /* VBOX_STRICT */
436
437
438
439	/**
440	* Updates the VM's last error record. If there was a VMX instruction error,
441	* reads the error data from the VMCS and updates VCPU's last error record as
442	* well.
443	*
444	* @param pVM Pointer to the VM.
445	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
446	* VERR_VMX_UNABLE_TO_START_VM or
447	* VERR_VMX_INVALID_VMCS_FIELD).
448	* @param rc The error code.
449	*/
450	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
451	{
452	AssertPtr(pVM);
453	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
454	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
455	{
456	AssertPtrReturnVoid(pVCpu);
457	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
458	}
459	pVM->hm.s.lLastError = rc;
460	}
461
462
463	/**
464	* Reads the VM-entry interruption-information field from the VMCS into the VMX
465	* transient structure.
466	*
467	* @returns VBox status code.
468	* @param pVmxTransient Pointer to the VMX transient structure.
469	*
470	* @remarks No-long-jump zone!!!
471	*/
472	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
473	{
474	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
475	AssertRCReturn(rc, rc);
476	return VINF_SUCCESS;
477	}
478
479
480	/**
481	* Reads the VM-entry exception error code field from the VMCS into
482	* the VMX transient structure.
483	*
484	* @returns VBox status code.
485	* @param pVmxTransient Pointer to the VMX transient structure.
486	*
487	* @remarks No-long-jump zone!!!
488	*/
489	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
490	{
491	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
492	AssertRCReturn(rc, rc);
493	return VINF_SUCCESS;
494	}
495
496
497	/**
498	* Reads the VM-entry exception error code field from the VMCS into
499	* the VMX transient structure.
500	*
501	* @returns VBox status code.
502	* @param pVCpu Pointer to the VMCPU.
503	* @param pVmxTransient Pointer to the VMX transient structure.
504	*
505	* @remarks No-long-jump zone!!!
506	*/
507	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
508	{
509	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
510	AssertRCReturn(rc, rc);
511	return VINF_SUCCESS;
512	}
513
514
515	/**
516	* Reads the VM-exit interruption-information field from the VMCS into the VMX
517	* transient structure.
518	*
519	* @returns VBox status code.
520	* @param pVCpu Pointer to the VMCPU.
521	* @param pVmxTransient Pointer to the VMX transient structure.
522	*/
523	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
524	{
525	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
526	{
527	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
528	AssertRCReturn(rc, rc);
529	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
530	}
531	return VINF_SUCCESS;
532	}
533
534
535	/**
536	* Reads the VM-exit interruption error code from the VMCS into the VMX
537	* transient structure.
538	*
539	* @returns VBox status code.
540	* @param pVCpu Pointer to the VMCPU.
541	* @param pVmxTransient Pointer to the VMX transient structure.
542	*/
543	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
544	{
545	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
546	{
547	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
548	AssertRCReturn(rc, rc);
549	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
550	}
551	return VINF_SUCCESS;
552	}
553
554
555	/**
556	* Reads the VM-exit instruction length field from the VMCS into the VMX
557	* transient structure.
558	*
559	* @returns VBox status code.
560	* @param pVCpu Pointer to the VMCPU.
561	* @param pVmxTransient Pointer to the VMX transient structure.
562	*/
563	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
564	{
565	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
566	{
567	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
568	AssertRCReturn(rc, rc);
569	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
570	}
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the exit qualification from the VMCS into the VMX transient structure.
577	*
578	* @returns VBox status code.
579	* @param pVCpu Pointer to the VMCPU.
580	* @param pVmxTransient Pointer to the VMX transient structure.
581	*/
582	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583	{
584	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
585	{
586	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
587	AssertRCReturn(rc, rc);
588	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
589	}
590	return VINF_SUCCESS;
591	}
592
593
594	/**
595	* Reads the IDT-vectoring information field from the VMCS into the VMX
596	* transient structure.
597	*
598	* @returns VBox status code.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*
601	* @remarks No-long-jump zone!!!
602	*/
603	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the IDT-vectoring error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Enters VMX root mode operation on the current CPU.
636	*
637	* @returns VBox status code.
638	* @param pVM Pointer to the VM (optional, can be NULL, after
639	* a resume).
640	* @param HCPhysCpuPage Physical address of the VMXON region.
641	* @param pvCpuPage Pointer to the VMXON region.
642	*/
643	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
644	{
645	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
646	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
648
649	if (pVM)
650	{
651	/* Write the VMCS revision dword to the VMXON region. */
652	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
653	}
654
655	/* Enable the VMX bit in CR4 if necessary. */
656	RTCCUINTREG uCr4 = ASMGetCR4();
657	if (!(uCr4 & X86_CR4_VMXE))
658	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
659
660	/* Enter VMX root mode. */
661	int rc = VMXEnable(HCPhysCpuPage);
662	if (RT_FAILURE(rc))
663	ASMSetCR4(uCr4);
664
665	return rc;
666	}
667
668
669	/**
670	* Exits VMX root mode operation on the current CPU.
671	*
672	* @returns VBox status code.
673	*/
674	static int hmR0VmxLeaveRootMode(void)
675	{
676	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
677
678	/* If we're for some reason not in VMX root mode, then don't leave it. */
679	if (ASMGetCR4() & X86_CR4_VMXE)
680	{
681	/* Exit VMX root mode and clear the VMX bit in CR4 */
682	VMXDisable();
683	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
684	return VINF_SUCCESS;
685	}
686
687	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
688	}
689
690
691	/**
692	* Allocates and maps one physically contiguous page. The allocated page is
693	* zero'd out. (Used by various VT-x structures).
694	*
695	* @returns IPRT status code.
696	* @param pMemObj Pointer to the ring-0 memory object.
697	* @param ppVirt Where to store the virtual address of the
698	* allocation.
699	* @param pPhys Where to store the physical address of the
700	* allocation.
701	*/
702	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
703	{
704	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
705	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
706	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
707
708	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
709	if (RT_FAILURE(rc))
710	return rc;
711	ppVirt = RTR0MemObjAddress(pMemObj);
712	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
713	ASMMemZero32(*ppVirt, PAGE_SIZE);
714	return VINF_SUCCESS;
715	}
716
717
718	/**
719	* Frees and unmaps an allocated physical page.
720	*
721	* @param pMemObj Pointer to the ring-0 memory object.
722	* @param ppVirt Where to re-initialize the virtual address of
723	* allocation as 0.
724	* @param pHCPhys Where to re-initialize the physical address of the
725	* allocation as 0.
726	*/
727	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
728	{
729	AssertPtr(pMemObj);
730	AssertPtr(ppVirt);
731	AssertPtr(pHCPhys);
732	if (*pMemObj != NIL_RTR0MEMOBJ)
733	{
734	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
735	AssertRC(rc);
736	*pMemObj = NIL_RTR0MEMOBJ;
737	*ppVirt = 0;
738	*pHCPhys = 0;
739	}
740	}
741
742
743	/**
744	* Worker function to free VT-x related structures.
745	*
746	* @returns IPRT status code.
747	* @param pVM Pointer to the VM.
748	*/
749	static void hmR0VmxStructsFree(PVM pVM)
750	{
751	for (VMCPUID i = 0; i < pVM->cCpus; i++)
752	{
753	PVMCPU pVCpu = &pVM->aCpus[i];
754	AssertPtr(pVCpu);
755
756	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
757	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
758	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
759	#endif
760
761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
763
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
765	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
766	}
767
768	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
769	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
771	#endif
772	}
773
774
775	/**
776	* Worker function to allocate VT-x related VM structures.
777	*
778	* @returns IPRT status code.
779	* @param pVM Pointer to the VM.
780	*/
781	static int hmR0VmxStructsAlloc(PVM pVM)
782	{
783	/*
784	* Initialize members up-front so we can cleanup properly on allocation failure.
785	*/
786	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
787	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
788	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
789	pVM->hm.s.vmx.HCPhys##a_Name = 0;
790
791	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
792	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
793	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
794	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
795
796	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
797	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
798	#endif
799	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
800
801	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
802	for (VMCPUID i = 0; i < pVM->cCpus; i++)
803	{
804	PVMCPU pVCpu = &pVM->aCpus[i];
805	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
806	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
808	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
810	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
811	#endif
812	}
813	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
814	#undef VMXLOCAL_INIT_VM_MEMOBJ
815
816	/*
817	* Allocate all the VT-x structures.
818	*/
819	int rc = VINF_SUCCESS;
820	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
822	if (RT_FAILURE(rc))
823	goto cleanup;
824	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
825	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
826	#endif
827
828	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
829	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
830	{
831	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
832	&pVM->hm.s.vmx.HCPhysApicAccess);
833	if (RT_FAILURE(rc))
834	goto cleanup;
835	}
836
837	/*
838	* Initialize per-VCPU VT-x structures.
839	*/
840	for (VMCPUID i = 0; i < pVM->cCpus; i++)
841	{
842	PVMCPU pVCpu = &pVM->aCpus[i];
843	AssertPtr(pVCpu);
844
845	/* Allocate the VM control structure (VMCS). */
846	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
847	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
848	if (RT_FAILURE(rc))
849	goto cleanup;
850
851	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
852	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
853	{
854	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
855	&pVCpu->hm.s.vmx.HCPhysVirtApic);
856	if (RT_FAILURE(rc))
857	goto cleanup;
858	}
859
860	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
861	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
862	{
863	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
864	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
865	if (RT_FAILURE(rc))
866	goto cleanup;
867	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
868	}
869
870	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
871	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
872	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873	if (RT_FAILURE(rc))
874	goto cleanup;
875
876	/* Allocate the VM-exit MSR-load page for the host MSRs. */
877	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
878	if (RT_FAILURE(rc))
879	goto cleanup;
880	#endif
881	}
882
883	return VINF_SUCCESS;
884
885	cleanup:
886	hmR0VmxStructsFree(pVM);
887	return rc;
888	}
889
890
891	/**
892	* Does global VT-x initialization (called during module initialization).
893	*
894	* @returns VBox status code.
895	*/
896	VMMR0DECL(int) VMXR0GlobalInit(void)
897	{
898	#ifdef HMVMX_USE_FUNCTION_TABLE
899	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
900	# ifdef VBOX_STRICT
901	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
902	Assert(g_apfnVMExitHandlers[i]);
903	# endif
904	#endif
905	return VINF_SUCCESS;
906	}
907
908
909	/**
910	* Does global VT-x termination (called during module termination).
911	*/
912	VMMR0DECL(void) VMXR0GlobalTerm()
913	{
914	/* Nothing to do currently. */
915	}
916
917
918	/**
919	* Sets up and activates VT-x on the current CPU.
920	*
921	* @returns VBox status code.
922	* @param pCpu Pointer to the global CPU info struct.
923	* @param pVM Pointer to the VM (can be NULL after a host resume
924	* operation).
925	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
926	* fEnabledByHost is true).
927	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
928	* @a fEnabledByHost is true).
929	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
930	* enable VT-x/AMD-V on the host.
931	*/
932	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
933	{
934	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
935	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
936
937	if (!fEnabledByHost)
938	{
939	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
940	if (RT_FAILURE(rc))
941	return rc;
942	}
943
944	/*
945	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
946	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
947	* each time while reusing a VPID after hitting the MaxASID limit once.
948	*/
949	if ( pVM
950	&& pVM->hm.s.vmx.fVpid
951	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
952	{
953	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
954	pCpu->fFlushAsidBeforeUse = false;
955	}
956	else
957	pCpu->fFlushAsidBeforeUse = true;
958
959	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
960	++pCpu->cTlbFlushes;
961
962	return VINF_SUCCESS;
963	}
964
965
966	/**
967	* Deactivates VT-x on the current CPU.
968	*
969	* @returns VBox status code.
970	* @param pCpu Pointer to the global CPU info struct.
971	* @param pvCpuPage Pointer to the VMXON region.
972	* @param HCPhysCpuPage Physical address of the VMXON region.
973	*/
974	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
975	{
976	NOREF(pCpu);
977	NOREF(pvCpuPage);
978	NOREF(HCPhysCpuPage);
979
980	return hmR0VmxLeaveRootMode();
981	}
982
983
984	/**
985	* Sets the permission bits for the specified MSR in the MSR bitmap.
986	*
987	* @param pVCpu Pointer to the VMCPU.
988	* @param uMSR The MSR value.
989	* @param enmRead Whether reading this MSR causes a VM-exit.
990	* @param enmWrite Whether writing this MSR causes a VM-exit.
991	*/
992	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
993	{
994	int32_t iBit;
995	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
996
997	/*
998	* Layout:
999	* 0x000 - 0x3ff - Low MSR read bits
1000	* 0x400 - 0x7ff - High MSR read bits
1001	* 0x800 - 0xbff - Low MSR write bits
1002	* 0xc00 - 0xfff - High MSR write bits
1003	*/
1004	if (uMsr <= 0x00001FFF)
1005	iBit = uMsr;
1006	else if ( uMsr >= 0xC0000000
1007	&& uMsr <= 0xC0001FFF)
1008	{
1009	iBit = (uMsr - 0xC0000000);
1010	pbMsrBitmap += 0x400;
1011	}
1012	else
1013	{
1014	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1015	return;
1016	}
1017
1018	Assert(iBit <= 0x1fff);
1019	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1020	ASMBitSet(pbMsrBitmap, iBit);
1021	else
1022	ASMBitClear(pbMsrBitmap, iBit);
1023
1024	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1025	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1026	else
1027	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1028	}
1029
1030
1031	/**
1032	* Flushes the TLB using EPT.
1033	*
1034	* @returns VBox status code.
1035	* @param pVM Pointer to the VM.
1036	* @param pVCpu Pointer to the VMCPU.
1037	* @param enmFlush Type of flush.
1038	*/
1039	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1040	{
1041	AssertPtr(pVM);
1042	Assert(pVM->hm.s.fNestedPaging);
1043
1044	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1045
1046	uint64_t descriptor[2];
1047	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1048	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1049
1050	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1051	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1052	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1053	}
1054
1055
1056	/**
1057	* Flushes the TLB using VPID.
1058	*
1059	* @returns VBox status code.
1060	* @param pVM Pointer to the VM.
1061	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1062	* enmFlush).
1063	* @param enmFlush Type of flush.
1064	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1065	* on @a enmFlush).
1066	*/
1067	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1068	{
1069	AssertPtr(pVM);
1070	Assert(pVM->hm.s.vmx.fVpid);
1071
1072	uint64_t descriptor[2];
1073	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1074	{
1075	descriptor[0] = 0;
1076	descriptor[1] = 0;
1077	}
1078	else
1079	{
1080	AssertPtr(pVCpu);
1081	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1082	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1083	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1084	descriptor[1] = GCPtr;
1085	}
1086
1087	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1088	AssertMsg(rc == VINF_SUCCESS,
1089	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1090	if ( RT_SUCCESS(rc)
1091	&& pVCpu)
1092	{
1093	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1094	}
1095	}
1096
1097
1098	/**
1099	* Invalidates a guest page by guest virtual address. Only relevant for
1100	* EPT/VPID, otherwise there is nothing really to invalidate.
1101	*
1102	* @returns VBox status code.
1103	* @param pVM Pointer to the VM.
1104	* @param pVCpu Pointer to the VMCPU.
1105	* @param GCVirt Guest virtual address of the page to invalidate.
1106	*/
1107	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1108	{
1109	AssertPtr(pVM);
1110	AssertPtr(pVCpu);
1111	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1112
1113	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1114	if (!fFlushPending)
1115	{
1116	/*
1117	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1118	* See @bugref{6043} and @bugref{6177}.
1119	*
1120	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1121	* function maybe called in a loop with individual addresses.
1122	*/
1123	if (pVM->hm.s.vmx.fVpid)
1124	{
1125	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1126	{
1127	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1128	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1129	}
1130	else
1131	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1132	}
1133	else if (pVM->hm.s.fNestedPaging)
1134	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1135	}
1136
1137	return VINF_SUCCESS;
1138	}
1139
1140
1141	/**
1142	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1143	* otherwise there is nothing really to invalidate.
1144	*
1145	* @returns VBox status code.
1146	* @param pVM Pointer to the VM.
1147	* @param pVCpu Pointer to the VMCPU.
1148	* @param GCPhys Guest physical address of the page to invalidate.
1149	*/
1150	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1151	{
1152	LogFlowFunc(("%RGp\n", GCPhys));
1153
1154	/*
1155	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1156	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1157	* This function might be called in a loop.
1158	*/
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1161	return VINF_SUCCESS;
1162	}
1163
1164
1165	/**
1166	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1167	* case where neither EPT nor VPID is supported by the CPU.
1168	*
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	*
1172	* @remarks Called with interrupts disabled.
1173	*/
1174	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1175	{
1176	NOREF(pVM);
1177	AssertPtr(pVCpu);
1178	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1179	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1180
1181	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1182	AssertPtr(pCpu);
1183
1184	pVCpu->hm.s.TlbShootdown.cPages = 0;
1185	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1186	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1187	pVCpu->hm.s.fForceTLBFlush = false;
1188	return;
1189	}
1190
1191
1192	/**
1193	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1194	*
1195	* @param pVM Pointer to the VM.
1196	* @param pVCpu Pointer to the VMCPU.
1197	* @remarks All references to "ASID" in this function pertains to "VPID" in
1198	* Intel's nomenclature. The reason is, to avoid confusion in compare
1199	* statements since the host-CPU copies are named "ASID".
1200	*
1201	* @remarks Called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1204	{
1205	AssertPtr(pVM);
1206	AssertPtr(pVCpu);
1207	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1208	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1209	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1210
1211	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1212	AssertPtr(pCpu);
1213
1214	/*
1215	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1216	* This can happen both for start & resume due to long jumps back to ring-3.
1217	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1218	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1219	*/
1220	bool fNewASID = false;
1221	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1222	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1223	{
1224	pVCpu->hm.s.fForceTLBFlush = true;
1225	fNewASID = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1227	}
1228
1229	/*
1230	* Check for explicit TLB shootdowns.
1231	*/
1232	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1233	{
1234	pVCpu->hm.s.fForceTLBFlush = true;
1235	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1236	}
1237
1238	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1239	if (pVCpu->hm.s.fForceTLBFlush)
1240	{
1241	if (fNewASID)
1242	{
1243	++pCpu->uCurrentAsid;
1244	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1245	{
1246	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1247	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1248	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1249	}
1250
1251	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1252	if (pCpu->fFlushAsidBeforeUse)
1253	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1254	}
1255	else
1256	{
1257	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1258	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1259	else
1260	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1261	}
1262
1263	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1264	pVCpu->hm.s.fForceTLBFlush = false;
1265	}
1266	else
1267	{
1268	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1269	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1270	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1271	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1272
1273	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1274	* not be executed. See hmQueueInvlPage() where it is commented
1275	* out. Support individual entry flushing someday. */
1276	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1277	{
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1279
1280	/*
1281	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1282	* as supported by the CPU.
1283	*/
1284	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1285	{
1286	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1287	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1288	}
1289	else
1290	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1291	}
1292	else
1293	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1294	}
1295	pVCpu->hm.s.TlbShootdown.cPages = 0;
1296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1297
1298	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1299	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1300	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1301	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1302	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1304
1305	/* Update VMCS with the VPID. */
1306	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1307	AssertRC(rc);
1308	}
1309
1310
1311	/**
1312	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1313	*
1314	* @returns VBox status code.
1315	* @param pVM Pointer to the VM.
1316	* @param pVCpu Pointer to the VMCPU.
1317	*
1318	* @remarks Called with interrupts disabled.
1319	*/
1320	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1321	{
1322	AssertPtr(pVM);
1323	AssertPtr(pVCpu);
1324	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1325	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1326
1327	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1328	AssertPtr(pCpu);
1329
1330	/*
1331	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1332	* This can happen both for start & resume due to long jumps back to ring-3.
1333	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1334	*/
1335	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1336	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1337	{
1338	pVCpu->hm.s.fForceTLBFlush = true;
1339	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1340	}
1341
1342	/* Check for explicit TLB shootdown flushes. */
1343	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1344	{
1345	pVCpu->hm.s.fForceTLBFlush = true;
1346	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1347	}
1348
1349	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1350	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1351
1352	if (pVCpu->hm.s.fForceTLBFlush)
1353	{
1354	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1355	pVCpu->hm.s.fForceTLBFlush = false;
1356	}
1357	else
1358	{
1359	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1360	* not be executed. See hmQueueInvlPage() where it is commented
1361	* out. Support individual entry flushing someday. */
1362	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1363	{
1364	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1365	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1366	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1367	}
1368	else
1369	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1370	}
1371
1372	pVCpu->hm.s.TlbShootdown.cPages = 0;
1373	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1374	}
1375
1376
1377	/**
1378	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1379	*
1380	* @returns VBox status code.
1381	* @param pVM Pointer to the VM.
1382	* @param pVCpu Pointer to the VMCPU.
1383	*
1384	* @remarks Called with interrupts disabled.
1385	*/
1386	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1387	{
1388	AssertPtr(pVM);
1389	AssertPtr(pVCpu);
1390	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1391	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1392
1393	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1394
1395	/*
1396	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1397	* This can happen both for start & resume due to long jumps back to ring-3.
1398	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1399	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1400	*/
1401	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1402	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1403	{
1404	pVCpu->hm.s.fForceTLBFlush = true;
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1406	}
1407
1408	/* Check for explicit TLB shootdown flushes. */
1409	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1410	{
1411	/*
1412	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1413	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1414	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1415	*/
1416	pVCpu->hm.s.fForceTLBFlush = true;
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1418	}
1419
1420	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1421	if (pVCpu->hm.s.fForceTLBFlush)
1422	{
1423	++pCpu->uCurrentAsid;
1424	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1425	{
1426	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1427	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1428	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1429	}
1430
1431	pVCpu->hm.s.fForceTLBFlush = false;
1432	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1433	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1434	if (pCpu->fFlushAsidBeforeUse)
1435	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1436	}
1437	else
1438	{
1439	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1440	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1441	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1442	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1443
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1450	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1451	{
1452	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1453	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1454	}
1455	else
1456	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1457	}
1458	else
1459	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1460	}
1461
1462	pVCpu->hm.s.TlbShootdown.cPages = 0;
1463	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1464
1465	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1466	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1467	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1468	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1469	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1471
1472	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1473	AssertRC(rc);
1474	}
1475
1476
1477	/**
1478	* Flushes the guest TLB entry based on CPU capabilities.
1479	*
1480	* @param pVCpu Pointer to the VMCPU.
1481	*/
1482	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1483	{
1484	PVM pVM = pVCpu->CTX_SUFF(pVM);
1485	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1486	{
1487	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1488	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1489	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1491	default:
1492	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1493	break;
1494	}
1495	}
1496
1497
1498	/**
1499	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1500	* TLB entries from the host TLB before VM-entry.
1501	*
1502	* @returns VBox status code.
1503	* @param pVM Pointer to the VM.
1504	*/
1505	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1506	{
1507	/*
1508	* Determine optimal flush type for Nested Paging.
1509	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1510	* guest execution (see hmR3InitFinalizeR0()).
1511	*/
1512	if (pVM->hm.s.fNestedPaging)
1513	{
1514	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1517	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1518	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1520	else
1521	{
1522	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1523	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1525	}
1526
1527	/* Make sure the write-back cacheable memory type for EPT is supported. */
1528	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1529	{
1530	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1531	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1532	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1533	}
1534	}
1535	else
1536	{
1537	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1538	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1539	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1540	}
1541	}
1542
1543	/*
1544	* Determine optimal flush type for VPID.
1545	*/
1546	if (pVM->hm.s.vmx.fVpid)
1547	{
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1552	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1554	else
1555	{
1556	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1557	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1558	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1561	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1562	pVM->hm.s.vmx.fVpid = false;
1563	}
1564	}
1565	else
1566	{
1567	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1568	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1569	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1570	pVM->hm.s.vmx.fVpid = false;
1571	}
1572	}
1573
1574	/*
1575	* Setup the handler for flushing tagged-TLBs.
1576	*/
1577	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1578	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1579	else if (pVM->hm.s.fNestedPaging)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1581	else if (pVM->hm.s.vmx.fVpid)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1583	else
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1585	return VINF_SUCCESS;
1586	}
1587
1588
1589	/**
1590	* Sets up pin-based VM-execution controls in the VMCS.
1591	*
1592	* @returns VBox status code.
1593	* @param pVM Pointer to the VM.
1594	* @param pVCpu Pointer to the VMCPU.
1595	*/
1596	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1597	{
1598	AssertPtr(pVM);
1599	AssertPtr(pVCpu);
1600
1601	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1602	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1603
1604	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1605	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1606	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1607
1608	/* Enable the VMX preemption timer. */
1609	if (pVM->hm.s.vmx.fUsePreemptTimer)
1610	{
1611	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1612	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1613	}
1614
1615	if ((val & zap) != val)
1616	{
1617	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1618	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1619	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1620	}
1621
1622	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1623	AssertRCReturn(rc, rc);
1624
1625	/* Update VCPU with the currently set pin-based VM-execution controls. */
1626	pVCpu->hm.s.vmx.u32PinCtls = val;
1627	return rc;
1628	}
1629
1630
1631	/**
1632	* Sets up processor-based VM-execution controls in the VMCS.
1633	*
1634	* @returns VBox status code.
1635	* @param pVM Pointer to the VM.
1636	* @param pVMCPU Pointer to the VMCPU.
1637	*/
1638	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1639	{
1640	AssertPtr(pVM);
1641	AssertPtr(pVCpu);
1642
1643	int rc = VERR_INTERNAL_ERROR_5;
1644	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1645	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1646
1647	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1648	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1649	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1654
1655	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1656	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1657	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1658	{
1659	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1660	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1661	}
1662
1663	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1664	if (!pVM->hm.s.fNestedPaging)
1665	{
1666	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1667	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1668	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1669	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1670	}
1671
1672	/* Use TPR shadowing if supported by the CPU. */
1673	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1674	{
1675	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1676	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1677	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1678	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1679	AssertRCReturn(rc, rc);
1680
1681	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1682	/* CR8 writes causes a VM-exit based on TPR threshold. */
1683	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1684	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1685	}
1686	else
1687	{
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1689	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1690	}
1691
1692	/* Use MSR-bitmaps if supported by the CPU. */
1693	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1694	{
1695	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1696
1697	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1698	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1699	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	AssertRCReturn(rc, rc);
1701
1702	/*
1703	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1704	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1705	*/
1706	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1707	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	}
1716
1717	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1720
1721	if ((val & zap) != val)
1722	{
1723	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1724	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1725	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1726	}
1727
1728	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1729	AssertRCReturn(rc, rc);
1730
1731	/* Update VCPU with the currently set processor-based VM-execution controls. */
1732	pVCpu->hm.s.vmx.u32ProcCtls = val;
1733
1734	/*
1735	* Secondary processor-based VM-execution controls.
1736	*/
1737	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1738	{
1739	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1740	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1741
1742	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1743	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1744
1745	if (pVM->hm.s.fNestedPaging)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1747	else
1748	{
1749	/*
1750	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1751	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1752	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1753	*/
1754	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1755	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1756	}
1757
1758	if (pVM->hm.s.vmx.fVpid)
1759	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1760
1761	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1763
1764	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1765	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1766	* done dynamically. */
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1768	{
1769	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1770	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1771	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1772	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1773	AssertRCReturn(rc, rc);
1774	}
1775
1776	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1777	{
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1779	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1780	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1781	}
1782
1783	if ((val & zap) != val)
1784	{
1785	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1786	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1787	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1788	}
1789
1790	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1791	AssertRCReturn(rc, rc);
1792
1793	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1794	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1795	}
1796
1797	return VINF_SUCCESS;
1798	}
1799
1800
1801	/**
1802	* Sets up miscellaneous (everything other than Pin & Processor-based
1803	* VM-execution) control fields in the VMCS.
1804	*
1805	* @returns VBox status code.
1806	* @param pVM Pointer to the VM.
1807	* @param pVCpu Pointer to the VMCPU.
1808	*/
1809	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1810	{
1811	AssertPtr(pVM);
1812	AssertPtr(pVCpu);
1813
1814	/** @todo Shouldn't we able to avoid initializing with 0? */
1815	int rc = VERR_GENERAL_FAILURE;
1816
1817	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1818	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1819	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1820
1821	/*
1822	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1823	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1824	* We thus use the exception bitmap to control it rather than use both.
1825	*/
1826	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1827	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1828
1829	/** @todo Explore possibility of using IO-bitmaps. */
1830	/* All IO & IOIO instructions cause VM-exits. */
1831	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1832	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1833
1834	/* Initialize the MSR-bitmap area. */
1835	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1836	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1837	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1838
1839	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1840	/* Setup MSR autoloading/storing. */
1841	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1843	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	AssertRCReturn(rc, rc);
1845	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1846	AssertRCReturn(rc, rc);
1847
1848	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1849	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1850	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1851	AssertRCReturn(rc, rc);
1852	#endif
1853
1854	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1855	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1856	AssertRCReturn(rc, rc);
1857
1858	/* Setup debug controls */
1859	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1860	AssertRCReturn(rc, rc);
1861	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1862	AssertRCReturn(rc, rc);
1863	return rc;
1864	}
1865
1866
1867	/**
1868	* Sets up the initial exception bitmap in the VMCS based on static conditions
1869	* (i.e. conditions that cannot ever change at runtime).
1870	*
1871	* @returns VBox status code.
1872	* @param pVM Pointer to the VM.
1873	* @param pVCpu Pointer to the VMCPU.
1874	*/
1875	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1876	{
1877	AssertPtr(pVM);
1878	AssertPtr(pVCpu);
1879
1880	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1881
1882	uint32_t u32XcptBitmap = 0;
1883
1884	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1885	if (!pVM->hm.s.fNestedPaging)
1886	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1887
1888	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1890	AssertRCReturn(rc, rc);
1891	return rc;
1892	}
1893
1894
1895	/**
1896	* Sets up the initial guest-state mask. The guest-state mask is consulted
1897	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1898	* for the nested virtualization case (as it would cause a VM-exit).
1899	*
1900	* @param pVCpu Pointer to the VMCPU.
1901	*/
1902	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1903	{
1904	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1905	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Does per-VM VT-x initialization.
1912	*
1913	* @returns VBox status code.
1914	* @param pVM Pointer to the VM.
1915	*/
1916	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1917	{
1918	LogFlowFunc(("pVM=%p\n", pVM));
1919
1920	int rc = hmR0VmxStructsAlloc(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	/**
1932	* Does per-VM VT-x termination.
1933	*
1934	* @returns VBox status code.
1935	* @param pVM Pointer to the VM.
1936	*/
1937	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1938	{
1939	LogFlowFunc(("pVM=%p\n", pVM));
1940
1941	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1942	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1943	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1944	#endif
1945	hmR0VmxStructsFree(pVM);
1946	return VINF_SUCCESS;
1947	}
1948
1949
1950	/**
1951	* Sets up the VM for execution under VT-x.
1952	* This function is only called once per-VM during initalization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1958	{
1959	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1961
1962	LogFlowFunc(("pVM=%p\n", pVM));
1963
1964	/*
1965	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1966	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1967	*/
1968	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
1969	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1970	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1971	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1972	{
1973	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1974	return VERR_INTERNAL_ERROR;
1975	}
1976
1977	/* Initialize these always, see hmR3InitFinalizeR0().*/
1978	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1979	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1980
1981	/* Setup the tagged-TLB flush handlers. */
1982	int rc = hmR0VmxSetupTaggedTlb(pVM);
1983	if (RT_FAILURE(rc))
1984	{
1985	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1986	return rc;
1987	}
1988
1989	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1990	{
1991	PVMCPU pVCpu = &pVM->aCpus[i];
1992	AssertPtr(pVCpu);
1993	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1994
1995	/* Set revision dword at the beginning of the VMCS structure. */
1996	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1997
1998	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1999	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2000	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2001	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2002
2003	/* Load this VMCS as the current VMCS. */
2004	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2017	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2018	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2019
2020	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2021	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2022	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2023
2024	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2025	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2026	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2027
2028	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2029	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2030	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2031	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2032	#endif
2033
2034	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2035	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2036	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2037	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2038
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2040	}
2041
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2048	* the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	* @param pVCpu Pointer to the VMCPU.
2053	*/
2054	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2055	{
2056	RTCCUINTREG uReg = ASMGetCR0();
2057	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2058	AssertRCReturn(rc, rc);
2059
2060	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2061	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2062	if (HMVMX_IS_64BIT_HOST_MODE())
2063	{
2064	uint64_t uRegCR3 = HMR0Get64bitCR3();
2065	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2066	}
2067	else
2068	#endif
2069	{
2070	uReg = ASMGetCR3();
2071	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2072	}
2073	AssertRCReturn(rc, rc);
2074
2075	uReg = ASMGetCR4();
2076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2077	AssertRCReturn(rc, rc);
2078	return rc;
2079	}
2080
2081
2082	/**
2083	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2084	* the host-state area in the VMCS.
2085	*
2086	* @returns VBox status code.
2087	* @param pVM Pointer to the VM.
2088	* @param pVCpu Pointer to the VMCPU.
2089	*/
2090	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2091	{
2092	int rc = VERR_INTERNAL_ERROR_5;
2093	RTSEL uSelDS = 0;
2094	RTSEL uSelES = 0;
2095	RTSEL uSelFS = 0;
2096	RTSEL uSelGS = 0;
2097	RTSEL uSelTR = 0;
2098
2099	/*
2100	* Host DS, ES, FS and GS segment registers.
2101	*/
2102	#if HC_ARCH_BITS == 64
2103	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2104	uSelDS = ASMGetDS();
2105	uSelES = ASMGetES();
2106	uSelFS = ASMGetFS();
2107	uSelGS = ASMGetGS();
2108	#endif
2109
2110	/*
2111	* Host CS and SS segment registers.
2112	*/
2113	RTSEL uSelCS;
2114	RTSEL uSelSS;
2115	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2116	if (HMVMX_IS_64BIT_HOST_MODE())
2117	{
2118	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2119	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2120	}
2121	else
2122	{
2123	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2124	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2125	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2126	}
2127	#else
2128	uSelCS = ASMGetCS();
2129	uSelSS = ASMGetSS();
2130	#endif
2131
2132	/*
2133	* Host TR segment register.
2134	*/
2135	uSelTR = ASMGetTR();
2136
2137	#if HC_ARCH_BITS == 64
2138	/*
2139	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2140	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2141	*/
2142	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2143	{
2144	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2145	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2146	uSelDS = 0;
2147	}
2148	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2149	{
2150	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2151	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2152	uSelES = 0;
2153	}
2154	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2155	{
2156	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2157	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2158	uSelFS = 0;
2159	}
2160	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2161	{
2162	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2163	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2164	uSelGS = 0;
2165	}
2166	#endif
2167
2168	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2169	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2170	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2171	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2172	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2173	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2174	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2175	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2176	Assert(uSelCS);
2177	Assert(uSelTR);
2178
2179	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2180	#if 0
2181	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2182	Assert(uSelSS != 0);
2183	#endif
2184
2185	/* Write these host selector fields into the host-state area in the VMCS. */
2186	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2187	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2188	#if HC_ARCH_BITS == 64
2189	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2190	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2191	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2192	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2193	#endif
2194	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2195
2196	/*
2197	* Host GDTR and IDTR.
2198	*/
2199	RTGDTR Gdtr;
2200	RT_ZERO(Gdtr);
2201	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2202	if (HMVMX_IS_64BIT_HOST_MODE())
2203	{
2204	X86XDTR64 Gdtr64;
2205	X86XDTR64 Idtr64;
2206	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2207	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2208	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2209
2210	Gdtr.cbGdt = Gdtr64.cb;
2211	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2212	}
2213	else
2214	#endif
2215	{
2216	RTIDTR Idtr;
2217	ASMGetGDTR(&Gdtr);
2218	ASMGetIDTR(&Idtr);
2219	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2220	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2221
2222	#if HC_ARCH_BITS == 64
2223	/*
2224	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2225	* maximum limit (0xffff) on every VM-exit.
2226	*/
2227	if (Gdtr.cbGdt != 0xffff)
2228	{
2229	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2230	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2231	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2232	}
2233
2234	/*
2235	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2236	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2237	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2238	*/
2239	if (Idtr.cbIdt < 0x0fff)
2240	{
2241	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2242	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2243	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2244	}
2245	#endif
2246	}
2247
2248	/*
2249	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2250	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2251	*/
2252	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2253	{
2254	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2255	return VERR_VMX_INVALID_HOST_STATE;
2256	}
2257
2258	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2259	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2260	if (HMVMX_IS_64BIT_HOST_MODE())
2261	{
2262	/* We need the 64-bit TR base for hybrid darwin. */
2263	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2264	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2265	}
2266	else
2267	#endif
2268	{
2269	uintptr_t uTRBase;
2270	#if HC_ARCH_BITS == 64
2271	uTRBase = X86DESC64_BASE(pDesc);
2272	#else
2273	uTRBase = X86DESC_BASE(pDesc);
2274	#endif
2275	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2276	}
2277	AssertRCReturn(rc, rc);
2278
2279	/*
2280	* Host FS base and GS base.
2281	*/
2282	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2283	if (HMVMX_IS_64BIT_HOST_MODE())
2284	{
2285	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2286	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2287	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2288	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2289
2290	# if HC_ARCH_BITS == 64
2291	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2292	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2293	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2294	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2295	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2296	# endif
2297	}
2298	#endif
2299	return rc;
2300	}
2301
2302
2303	/**
2304	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2305	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2306	* the host after every successful VM exit.
2307	*
2308	* @returns VBox status code.
2309	* @param pVM Pointer to the VM.
2310	* @param pVCpu Pointer to the VMCPU.
2311	*/
2312	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2313	{
2314	AssertPtr(pVCpu);
2315	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2316
2317	int rc = VINF_SUCCESS;
2318	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2319	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2320	uint32_t cHostMsrs = 0;
2321	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2322
2323	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2324	{
2325	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2326
2327	# if HC_ARCH_BITS == 64
2328	/* Paranoia. 64-bit code requires these bits to be set always. */
2329	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2330
2331	/*
2332	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2333	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2334	* some reason (e.g. allow transparent reads) we would activate the code below.
2335	*/
2336	# if 0
2337	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2338	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2339	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2340	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2341	if (CPUMIsGuestInLongMode(pVCpu))
2342	{
2343	uint64_t u64GuestEfer;
2344	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2345	AssertRC(rc);
2346
2347	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2348	{
2349	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2350	pHostMsr->u32Reserved = 0;
2351	pHostMsr->u64Value = u64HostEfer;
2352	pHostMsr++; cHostMsrs++;
2353	}
2354	}
2355	# endif
2356	# else /* HC_ARCH_BITS != 64 */
2357	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2358	pHostMsr->u32Reserved = 0;
2359	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2360	if (CPUMIsGuestInLongMode(pVCpu))
2361	{
2362	/* Must match the EFER value in our 64 bits switcher. */
2363	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2364	}
2365	else
2366	# endif
2367	pHostMsr->u64Value = u64HostEfer;
2368	pHostMsr++; cHostMsrs++;
2369	# endif /* HC_ARCH_BITS == 64 */
2370	}
2371
2372	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2373	if (HMVMX_IS_64BIT_HOST_MODE())
2374	{
2375	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2376	pHostMsr->u32Reserved = 0;
2377	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2378	pHostMsr++; cHostMsrs++;
2379	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2380	pHostMsr->u32Reserved = 0;
2381	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2382	pHostMsr++; cHostMsrs++;
2383	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2384	pHostMsr->u32Reserved = 0;
2385	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2386	pHostMsr++; cHostMsrs++;
2387	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2388	pHostMsr->u32Reserved = 0;
2389	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2390	pHostMsr++; cHostMsrs++;
2391	}
2392	# endif
2393
2394	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2395	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2396	{
2397	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2398	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2399	}
2400
2401	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2402	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2403
2404	/*
2405	* Host Sysenter MSRs.
2406	*/
2407	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2408	AssertRCReturn(rc, rc);
2409	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2410	if (HMVMX_IS_64BIT_HOST_MODE())
2411	{
2412	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2413	AssertRCReturn(rc, rc);
2414	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2415	}
2416	else
2417	{
2418	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2419	AssertRCReturn(rc, rc);
2420	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2421	}
2422	#elif HC_ARCH_BITS == 32
2423	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2424	AssertRCReturn(rc, rc);
2425	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2426	#else
2427	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2428	AssertRCReturn(rc, rc);
2429	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2430	#endif
2431	AssertRCReturn(rc, rc);
2432
2433	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2434	* hmR0VmxSetupExitCtls() !! */
2435	return rc;
2436	}
2437
2438
2439	/**
2440	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2441	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2442	* controls".
2443	*
2444	* @returns VBox status code.
2445	* @param pVCpu Pointer to the VMCPU.
2446	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2447	* out-of-sync. Make sure to update the required fields
2448	* before using them.
2449	*
2450	* @remarks No-long-jump zone!!!
2451	*/
2452	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2453	{
2454	int rc = VINF_SUCCESS;
2455	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2456	{
2457	PVM pVM = pVCpu->CTX_SUFF(pVM);
2458	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2459	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2460
2461	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2462	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2463
2464	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2465	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2466	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2467	else
2468	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2469
2470	/*
2471	* The following should not be set (since we're not in SMM mode):
2472	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2473	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2474	*/
2475
2476	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2477	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2478	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2479
2480	if ((val & zap) != val)
2481	{
2482	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2483	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2484	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2485	}
2486
2487	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2488	AssertRCReturn(rc, rc);
2489
2490	/* Update VCPU with the currently set VM-exit controls. */
2491	pVCpu->hm.s.vmx.u32EntryCtls = val;
2492	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2493	}
2494	return rc;
2495	}
2496
2497
2498	/**
2499	* Sets up the VM-exit controls in the VMCS.
2500	*
2501	* @returns VBox status code.
2502	* @param pVM Pointer to the VM.
2503	* @param pVCpu Pointer to the VMCPU.
2504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2505	* out-of-sync. Make sure to update the required fields
2506	* before using them.
2507	*
2508	* @remarks requires EFER.
2509	*/
2510	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2511	{
2512	int rc = VINF_SUCCESS;
2513	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2514	{
2515	PVM pVM = pVCpu->CTX_SUFF(pVM);
2516	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2517	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2518
2519	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2520	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2521
2522	/*
2523	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2524	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2525	*/
2526	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2527	if (HMVMX_IS_64BIT_HOST_MODE())
2528	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2529	else
2530	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2531	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2532	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2533	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2534	else
2535	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2536	#endif
2537
2538	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2539	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2540
2541	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2542	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2543	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2544	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2545	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2546
2547	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2548	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2549
2550	if ((val & zap) != val)
2551	{
2552	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2553	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2554	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2555	}
2556
2557	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2558	AssertRCReturn(rc, rc);
2559
2560	/* Update VCPU with the currently set VM-exit controls. */
2561	pVCpu->hm.s.vmx.u32ExitCtls = val;
2562	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2563	}
2564	return rc;
2565	}
2566
2567
2568	/**
2569	* Loads the guest APIC and related state.
2570	*
2571	* @returns VBox status code.
2572	* @param pVM Pointer to the VM.
2573	* @param pVCpu Pointer to the VMCPU.
2574	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2575	* out-of-sync. Make sure to update the required fields
2576	* before using them.
2577	*/
2578	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2579	{
2580	int rc = VINF_SUCCESS;
2581	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2582	{
2583	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2584	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2585	{
2586	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2587
2588	bool fPendingIntr = false;
2589	uint8_t u8Tpr = 0;
2590	uint8_t u8PendingIntr = 0;
2591	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2592	AssertRCReturn(rc, rc);
2593
2594	/*
2595	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2596	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2597	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2598	* the interrupt when we VM-exit for other reasons.
2599	*/
2600	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2601	uint32_t u32TprThreshold = 0;
2602	if (fPendingIntr)
2603	{
2604	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2605	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2606	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2607	if (u8PendingPriority <= u8TprPriority)
2608	u32TprThreshold = u8PendingPriority;
2609	else
2610	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2611	}
2612	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2613
2614	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2615	AssertRCReturn(rc, rc);
2616
2617	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2618	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2619	{
2620	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2621	pMixedCtx->msrLSTAR = u8Tpr;
2622	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2623	{
2624	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2625	if (fPendingIntr)
2626	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2627	else
2628	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2629	}
2630	}
2631	}
2632
2633	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2634	}
2635	return rc;
2636	}
2637
2638
2639	/**
2640	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2641	*
2642	* @returns
2643	* @param pVCpu Pointer to the VMCPU.
2644	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2645	* out-of-sync. Make sure to update the required fields
2646	* before using them.
2647	*
2648	* @remarks No-long-jump zone!!!
2649	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2650	*/
2651	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2652	{
2653	/*
2654	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2655	* inhibit interrupts or clear any existing interrupt-inhibition.
2656	*/
2657	uint32_t uIntrState = 0;
2658	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2659	{
2660	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2661	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2662	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2663	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2664	{
2665	/*
2666	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2667	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2668	*/
2669	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2670	}
2671	else if (pMixedCtx->eflags.Bits.u1IF)
2672	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2673	else
2674	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2675	}
2676	return uIntrState;
2677	}
2678
2679
2680	/**
2681	* Loads the guest's interruptibility-state into the guest-state area in the
2682	* VMCS.
2683	*
2684	* @returns VBox status code.
2685	* @param pVCpu Pointer to the VMCPU.
2686	* @param uIntrState The interruptibility-state to set.
2687	*/
2688	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2689	{
2690	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2691	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2692	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2693	AssertRCReturn(rc, rc);
2694	return rc;
2695	}
2696
2697
2698	/**
2699	* Loads the guest's RIP into the guest-state area in the VMCS.
2700	*
2701	* @returns VBox status code.
2702	* @param pVCpu Pointer to the VMCPU.
2703	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2704	* out-of-sync. Make sure to update the required fields
2705	* before using them.
2706	*
2707	* @remarks No-long-jump zone!!!
2708	*/
2709	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2710	{
2711	int rc = VINF_SUCCESS;
2712	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2713	{
2714	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2715	AssertRCReturn(rc, rc);
2716	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2717	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2718	}
2719	return rc;
2720	}
2721
2722
2723	/**
2724	* Loads the guest's RSP into the guest-state area in the VMCS.
2725	*
2726	* @returns VBox status code.
2727	* @param pVCpu Pointer to the VMCPU.
2728	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2729	* out-of-sync. Make sure to update the required fields
2730	* before using them.
2731	*
2732	* @remarks No-long-jump zone!!!
2733	*/
2734	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2735	{
2736	int rc = VINF_SUCCESS;
2737	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2738	{
2739	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2740	AssertRCReturn(rc, rc);
2741	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2742	}
2743	return rc;
2744	}
2745
2746
2747	/**
2748	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2749	*
2750	* @returns VBox status code.
2751	* @param pVCpu Pointer to the VMCPU.
2752	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2753	* out-of-sync. Make sure to update the required fields
2754	* before using them.
2755	*
2756	* @remarks No-long-jump zone!!!
2757	*/
2758	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2759	{
2760	int rc = VINF_SUCCESS;
2761	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2762	{
2763	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2764	Let us assert it as such and use 32-bit VMWRITE. */
2765	Assert(!(pMixedCtx->rflags.u64 >> 32));
2766	X86EFLAGS uEFlags = pMixedCtx->eflags;
2767	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2768	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2769
2770	/*
2771	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2772	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2773	*/
2774	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2775	{
2776	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2777	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2778	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2779	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2780	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2781	}
2782
2783	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2784	AssertRCReturn(rc, rc);
2785
2786	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2787	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2788	}
2789	return rc;
2790	}
2791
2792
2793	/**
2794	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2795	*
2796	* @returns VBox status code.
2797	* @param pVCpu Pointer to the VMCPU.
2798	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2799	* out-of-sync. Make sure to update the required fields
2800	* before using them.
2801	*
2802	* @remarks No-long-jump zone!!!
2803	*/
2804	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2805	{
2806	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2807	AssertRCReturn(rc, rc);
2808	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2809	AssertRCReturn(rc, rc);
2810	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2811	AssertRCReturn(rc, rc);
2812	return rc;
2813	}
2814
2815
2816	/**
2817	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2818	* in the VMCS.
2819	*
2820	* @returns VBox status code.
2821	* @param pVM Pointer to the VM.
2822	* @param pVCpu Pointer to the VMCPU.
2823	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2824	* out-of-sync. Make sure to update the required fields
2825	* before using them.
2826	*
2827	* @remarks No-long-jump zone!!!
2828	*/
2829	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2830	{
2831	int rc = VINF_SUCCESS;
2832	PVM pVM = pVCpu->CTX_SUFF(pVM);
2833
2834	/*
2835	* Guest CR0.
2836	* Guest FPU.
2837	*/
2838	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2839	{
2840	Assert(!(pCtx->cr0 >> 32));
2841	uint32_t u32GuestCR0 = pCtx->cr0;
2842
2843	/* The guest's view (read access) of its CR0 is unblemished. */
2844	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2845	AssertRCReturn(rc, rc);
2846	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2847
2848	/* Setup VT-x's view of the guest CR0. */
2849	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2850	if (pVM->hm.s.fNestedPaging)
2851	{
2852	if (CPUMIsGuestPagingEnabledEx(pCtx))
2853	{
2854	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2855	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2856	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2857	}
2858	else
2859	{
2860	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2861	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2862	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2863	}
2864
2865	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2866	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2867	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2868
2869	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2870	AssertRCReturn(rc, rc);
2871	}
2872	else
2873	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2874
2875	/*
2876	* Guest FPU bits.
2877	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2878	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2879	*/
2880	u32GuestCR0 \|= X86_CR0_NE;
2881	bool fInterceptNM = false;
2882	if (CPUMIsGuestFPUStateActive(pVCpu))
2883	{
2884	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2885	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2886	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2887	}
2888	else
2889	{
2890	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2891	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2892	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2893	}
2894
2895	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2896	bool fInterceptMF = false;
2897	if (!(pCtx->cr0 & X86_CR0_NE))
2898	fInterceptMF = true;
2899
2900	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2901	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2902	{
2903	Assert(PDMVmmDevHeapIsEnabled(pVM));
2904	Assert(pVM->hm.s.vmx.pRealModeTSS);
2905	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2906	fInterceptNM = true;
2907	fInterceptMF = true;
2908	}
2909	else
2910	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2911
2912	if (fInterceptNM)
2913	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2914	else
2915	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2916
2917	if (fInterceptMF)
2918	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2919	else
2920	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2921
2922	/* Additional intercepts for debugging, define these yourself explicitly. */
2923	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2924	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2925	\| RT_BIT(X86_XCPT_DB)
2926	\| RT_BIT(X86_XCPT_DE)
2927	\| RT_BIT(X86_XCPT_NM)
2928	\| RT_BIT(X86_XCPT_UD)
2929	\| RT_BIT(X86_XCPT_NP)
2930	\| RT_BIT(X86_XCPT_SS)
2931	\| RT_BIT(X86_XCPT_GP)
2932	\| RT_BIT(X86_XCPT_PF)
2933	\| RT_BIT(X86_XCPT_MF);
2934	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2935	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2936	#endif
2937
2938	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2939
2940	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2941	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2942	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2943	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2944	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2945	else
2946	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2947
2948	u32GuestCR0 \|= uSetCR0;
2949	u32GuestCR0 &= uZapCR0;
2950	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2951
2952	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2953	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2954	AssertRCReturn(rc, rc);
2955	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2956	AssertRCReturn(rc, rc);
2957	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2958
2959	/*
2960	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2961	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2962	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2963	*/
2964	uint32_t u32CR0Mask = 0;
2965	u32CR0Mask = X86_CR0_PE
2966	\| X86_CR0_NE
2967	\| X86_CR0_WP
2968	\| X86_CR0_PG
2969	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2970	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2971	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2972	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2973	u32CR0Mask &= ~X86_CR0_PE;
2974	if (pVM->hm.s.fNestedPaging)
2975	u32CR0Mask &= ~X86_CR0_WP;
2976
2977	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2978	if (fInterceptNM)
2979	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2980	else
2981	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2982
2983	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2984	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
2985	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2986	AssertRCReturn(rc, rc);
2987
2988	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2989	}
2990
2991	/*
2992	* Guest CR2.
2993	* It's always loaded in the assembler code. Nothing to do here.
2994	*/
2995
2996	/*
2997	* Guest CR3.
2998	*/
2999	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3000	{
3001	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3002	if (pVM->hm.s.fNestedPaging)
3003	{
3004	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3005
3006	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3007	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3008	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3009	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3010
3011	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3012	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3013	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3014
3015	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3016	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3017	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3018	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3019
3020	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3021	AssertRCReturn(rc, rc);
3022	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3023
3024	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3025	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3026	{
3027	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3028	if (CPUMIsGuestInPAEModeEx(pCtx))
3029	{
3030	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3031	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3033	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3034	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3035	}
3036
3037	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3038	have Unrestricted Execution to handle the guest when it's not using paging. */
3039	GCPhysGuestCR3 = pCtx->cr3;
3040	}
3041	else
3042	{
3043	/*
3044	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3045	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3046	* EPT takes care of translating it to host-physical addresses.
3047	*/
3048	RTGCPHYS GCPhys;
3049	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3050	Assert(PDMVmmDevHeapIsEnabled(pVM));
3051
3052	/* We obtain it here every time as the guest could have relocated this PCI region. */
3053	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3054	AssertRCReturn(rc, rc);
3055
3056	GCPhysGuestCR3 = GCPhys;
3057	}
3058
3059	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3060	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3061	}
3062	else
3063	{
3064	/* Non-nested paging case, just use the hypervisor's CR3. */
3065	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3066
3067	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3068	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3069	}
3070	AssertRCReturn(rc, rc);
3071
3072	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3073	}
3074
3075	/*
3076	* Guest CR4.
3077	*/
3078	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3079	{
3080	Assert(!(pCtx->cr4 >> 32));
3081	uint32_t u32GuestCR4 = pCtx->cr4;
3082
3083	/* The guest's view of its CR4 is unblemished. */
3084	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3085	AssertRCReturn(rc, rc);
3086	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3087
3088	/* Setup VT-x's view of the guest CR4. */
3089	/*
3090	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3091	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3092	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3093	*/
3094	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3095	{
3096	Assert(pVM->hm.s.vmx.pRealModeTSS);
3097	Assert(PDMVmmDevHeapIsEnabled(pVM));
3098	u32GuestCR4 &= ~X86_CR4_VME;
3099	}
3100
3101	if (pVM->hm.s.fNestedPaging)
3102	{
3103	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3104	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3105	{
3106	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3107	u32GuestCR4 \|= X86_CR4_PSE;
3108	/* Our identity mapping is a 32 bits page directory. */
3109	u32GuestCR4 &= ~X86_CR4_PAE;
3110	}
3111	/* else use guest CR4.*/
3112	}
3113	else
3114	{
3115	/*
3116	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3117	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3118	*/
3119	switch (pVCpu->hm.s.enmShadowMode)
3120	{
3121	case PGMMODE_REAL: /* Real-mode. */
3122	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3123	case PGMMODE_32_BIT: /* 32-bit paging. */
3124	{
3125	u32GuestCR4 &= ~X86_CR4_PAE;
3126	break;
3127	}
3128
3129	case PGMMODE_PAE: /* PAE paging. */
3130	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3131	{
3132	u32GuestCR4 \|= X86_CR4_PAE;
3133	break;
3134	}
3135
3136	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3137	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3138	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3139	break;
3140	#endif
3141	default:
3142	AssertFailed();
3143	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3144	}
3145	}
3146
3147	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3148	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3149	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3150	u32GuestCR4 \|= uSetCR4;
3151	u32GuestCR4 &= uZapCR4;
3152
3153	/* Write VT-x's view of the guest CR4 into the VMCS. */
3154	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3155	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3156	AssertRCReturn(rc, rc);
3157
3158	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3159	uint32_t u32CR4Mask = 0;
3160	u32CR4Mask = X86_CR4_VME
3161	\| X86_CR4_PAE
3162	\| X86_CR4_PGE
3163	\| X86_CR4_PSE
3164	\| X86_CR4_VMXE;
3165	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3166	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3167	AssertRCReturn(rc, rc);
3168
3169	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3170	}
3171	return rc;
3172	}
3173
3174
3175	/**
3176	* Loads the guest debug registers into the guest-state area in the VMCS.
3177	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3178	*
3179	* @returns VBox status code.
3180	* @param pVCpu Pointer to the VMCPU.
3181	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3182	* out-of-sync. Make sure to update the required fields
3183	* before using them.
3184	*
3185	* @remarks No-long-jump zone!!!
3186	*/
3187	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3188	{
3189	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3190	return VINF_SUCCESS;
3191
3192	#ifdef VBOX_STRICT
3193	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3194	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3195	{
3196	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3197	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3198	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3199	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3200	}
3201	#endif
3202
3203	int rc = VERR_INTERNAL_ERROR_5;
3204	PVM pVM = pVCpu->CTX_SUFF(pVM);
3205	bool fInterceptDB = false;
3206	bool fInterceptMovDRx = false;
3207	if (DBGFIsStepping(pVCpu))
3208	{
3209	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3210	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3211	{
3212	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3213	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3214	AssertRCReturn(rc, rc);
3215	Assert(fInterceptDB == false);
3216	}
3217	else
3218	{
3219	fInterceptDB = true;
3220	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3221	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3222	}
3223	}
3224
3225	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3226	{
3227	if (!CPUMIsHyperDebugStateActive(pVCpu))
3228	{
3229	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3230	AssertRC(rc);
3231	}
3232	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3233	fInterceptMovDRx = true;
3234	}
3235	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3236	{
3237	if (!CPUMIsGuestDebugStateActive(pVCpu))
3238	{
3239	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3240	AssertRC(rc);
3241	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3242	}
3243	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3244	Assert(fInterceptMovDRx == false);
3245	}
3246	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3247	{
3248	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3249	fInterceptMovDRx = true;
3250	}
3251
3252	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3253	if (fInterceptDB)
3254	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3255	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3256	{
3257	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3258	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3259	#endif
3260	}
3261
3262	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3263	if (fInterceptMovDRx)
3264	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3265	else
3266	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3267
3268	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3269	AssertRCReturn(rc, rc);
3270	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3271	AssertRCReturn(rc, rc);
3272
3273	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3274	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3275	AssertRCReturn(rc, rc);
3276
3277	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3278	return rc;
3279	}
3280
3281
3282	#ifdef VBOX_STRICT
3283	/**
3284	* Strict function to validate segment registers.
3285	*
3286	* @remarks Requires CR0.
3287	*/
3288	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3289	{
3290	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3291	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3292	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3293
3294	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3295	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3296	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3297	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3298	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3299	{
3300	/* Protected mode checks */
3301	/* CS */
3302	Assert(pCtx->cs.Attr.n.u1Present);
3303	Assert(!(pCtx->cs.Attr.u & 0xf00));
3304	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3305	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3306	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3307	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3308	\|\| (pCtx->cs.Attr.n.u1Granularity));
3309	/* CS cannot be loaded with NULL in protected mode. */
3310	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3311	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3312	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3313	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3314	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3315	else
3316	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3317	/* SS */
3318	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3319	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3320	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3321	if ( !(pCtx->cr0 & X86_CR0_PE)
3322	\|\| pCtx->cs.Attr.n.u4Type == 3)
3323	{
3324	Assert(!pCtx->ss.Attr.n.u2Dpl);
3325	}
3326	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3327	{
3328	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3329	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3330	Assert(pCtx->ss.Attr.n.u1Present);
3331	Assert(!(pCtx->ss.Attr.u & 0xf00));
3332	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3333	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3334	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3335	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3336	\|\| (pCtx->ss.Attr.n.u1Granularity));
3337	}
3338	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3339	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3340	{
3341	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3342	Assert(pCtx->ds.Attr.n.u1Present);
3343	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3344	Assert(!(pCtx->ds.Attr.u & 0xf00));
3345	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3346	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3347	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3348	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3349	\|\| (pCtx->ds.Attr.n.u1Granularity));
3350	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3351	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3352	}
3353	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3354	{
3355	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3356	Assert(pCtx->es.Attr.n.u1Present);
3357	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3358	Assert(!(pCtx->es.Attr.u & 0xf00));
3359	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3360	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3361	\|\| !(pCtx->es.Attr.n.u1Granularity));
3362	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3363	\|\| (pCtx->es.Attr.n.u1Granularity));
3364	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3365	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3366	}
3367	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3368	{
3369	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3370	Assert(pCtx->fs.Attr.n.u1Present);
3371	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3372	Assert(!(pCtx->fs.Attr.u & 0xf00));
3373	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3374	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3375	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3376	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3377	\|\| (pCtx->fs.Attr.n.u1Granularity));
3378	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3379	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3380	}
3381	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3382	{
3383	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3384	Assert(pCtx->gs.Attr.n.u1Present);
3385	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3386	Assert(!(pCtx->gs.Attr.u & 0xf00));
3387	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3388	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3389	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3390	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3391	\|\| (pCtx->gs.Attr.n.u1Granularity));
3392	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3393	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3394	}
3395	/* 64-bit capable CPUs. */
3396	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3397	Assert(!(pCtx->cs.u64Base >> 32));
3398	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3399	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3400	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3401	# endif
3402	}
3403	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3404	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3405	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3406	{
3407	/* Real and v86 mode checks. */
3408	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3409	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3410	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3411	{
3412	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3413	}
3414	else
3415	{
3416	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3417	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3418	}
3419
3420	/* CS */
3421	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3422	Assert(pCtx->cs.u32Limit == 0xffff);
3423	Assert(u32CSAttr == 0xf3);
3424	/* SS */
3425	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3426	Assert(pCtx->ss.u32Limit == 0xffff);
3427	Assert(u32SSAttr == 0xf3);
3428	/* DS */
3429	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3430	Assert(pCtx->ds.u32Limit == 0xffff);
3431	Assert(u32DSAttr == 0xf3);
3432	/* ES */
3433	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3434	Assert(pCtx->es.u32Limit == 0xffff);
3435	Assert(u32ESAttr == 0xf3);
3436	/* FS */
3437	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3438	Assert(pCtx->fs.u32Limit == 0xffff);
3439	Assert(u32FSAttr == 0xf3);
3440	/* GS */
3441	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3442	Assert(pCtx->gs.u32Limit == 0xffff);
3443	Assert(u32GSAttr == 0xf3);
3444	/* 64-bit capable CPUs. */
3445	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3446	Assert(!(pCtx->cs.u64Base >> 32));
3447	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3448	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3449	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3450	# endif
3451	}
3452	}
3453	#endif /* VBOX_STRICT */
3454
3455
3456	/**
3457	* Writes a guest segment register into the guest-state area in the VMCS.
3458	*
3459	* @returns VBox status code.
3460	* @param pVCpu Pointer to the VMCPU.
3461	* @param idxSel Index of the selector in the VMCS.
3462	* @param idxLimit Index of the segment limit in the VMCS.
3463	* @param idxBase Index of the segment base in the VMCS.
3464	* @param idxAccess Index of the access rights of the segment in the VMCS.
3465	* @param pSelReg Pointer to the segment selector.
3466	* @param pCtx Pointer to the guest-CPU context.
3467	*
3468	* @remarks No-long-jump zone!!!
3469	*/
3470	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3471	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3472	{
3473	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3474	AssertRCReturn(rc, rc);
3475	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3476	AssertRCReturn(rc, rc);
3477	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3478	AssertRCReturn(rc, rc);
3479
3480	uint32_t u32Access = pSelReg->Attr.u;
3481	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3482	{
3483	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3484	u32Access = 0xf3;
3485	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3486	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3487	}
3488	else
3489	{
3490	/*
3491	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3492	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3493	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3494	* loaded in protected-mode have their attribute as 0.
3495	*/
3496	if (!u32Access)
3497	u32Access = HMVMX_SEL_UNUSABLE;
3498	}
3499
3500	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3501	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3502	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3503
3504	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3505	AssertRCReturn(rc, rc);
3506	return rc;
3507	}
3508
3509
3510	/**
3511	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3512	* into the guest-state area in the VMCS.
3513	*
3514	* @returns VBox status code.
3515	* @param pVM Pointer to the VM.
3516	* @param pVCPU Pointer to the VMCPU.
3517	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3518	* out-of-sync. Make sure to update the required fields
3519	* before using them.
3520	*
3521	* @remarks Requires CR0 (strict builds validation).
3522	* @remarks No-long-jump zone!!!
3523	*/
3524	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3525	{
3526	int rc = VERR_INTERNAL_ERROR_5;
3527	PVM pVM = pVCpu->CTX_SUFF(pVM);
3528
3529	/*
3530	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3531	*/
3532	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3533	{
3534	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3535	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3536	{
3537	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3538	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3539	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3540	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3541	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3542	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3543	}
3544
3545	#ifdef VBOX_WITH_REM
3546	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3547	{
3548	Assert(pVM->hm.s.vmx.pRealModeTSS);
3549	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3550	if ( pVCpu->hm.s.vmx.fWasInRealMode
3551	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3552	{
3553	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3554	in real-mode (e.g. OpenBSD 4.0) */
3555	REMFlushTBs(pVM);
3556	Log4(("Load: Switch to protected mode detected!\n"));
3557	pVCpu->hm.s.vmx.fWasInRealMode = false;
3558	}
3559	}
3560	#endif
3561	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3562	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3563	AssertRCReturn(rc, rc);
3564	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3565	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3566	AssertRCReturn(rc, rc);
3567	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3568	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3569	AssertRCReturn(rc, rc);
3570	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3571	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3572	AssertRCReturn(rc, rc);
3573	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3574	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3575	AssertRCReturn(rc, rc);
3576	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3577	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3578	AssertRCReturn(rc, rc);
3579
3580	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3581	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3582	#ifdef VBOX_STRICT
3583	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3584	#endif
3585	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3586	}
3587
3588	/*
3589	* Guest TR.
3590	*/
3591	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3592	{
3593	/*
3594	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3595	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3596	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3597	*/
3598	uint16_t u16Sel = 0;
3599	uint32_t u32Limit = 0;
3600	uint64_t u64Base = 0;
3601	uint32_t u32AccessRights = 0;
3602
3603	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3604	{
3605	u16Sel = pMixedCtx->tr.Sel;
3606	u32Limit = pMixedCtx->tr.u32Limit;
3607	u64Base = pMixedCtx->tr.u64Base;
3608	u32AccessRights = pMixedCtx->tr.Attr.u;
3609	}
3610	else
3611	{
3612	Assert(pVM->hm.s.vmx.pRealModeTSS);
3613	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3614
3615	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3616	RTGCPHYS GCPhys;
3617	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3618	AssertRCReturn(rc, rc);
3619
3620	X86DESCATTR DescAttr;
3621	DescAttr.u = 0;
3622	DescAttr.n.u1Present = 1;
3623	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3624
3625	u16Sel = 0;
3626	u32Limit = HM_VTX_TSS_SIZE;
3627	u64Base = GCPhys; /* in real-mode phys = virt. */
3628	u32AccessRights = DescAttr.u;
3629	}
3630
3631	/* Validate. */
3632	Assert(!(u16Sel & RT_BIT(2)));
3633	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3634	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3635	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3636	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3637	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3638	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3639	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3640	Assert( (u32Limit & 0xfff) == 0xfff
3641	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3642	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3643	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3644
3645	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3646	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3647	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3648	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3649
3650	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3651	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3652	}
3653
3654	/*
3655	* Guest GDTR.
3656	*/
3657	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3658	{
3659	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3660	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3661
3662	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3663	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3664	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3665	}
3666
3667	/*
3668	* Guest LDTR.
3669	*/
3670	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3671	{
3672	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3673	uint32_t u32Access = 0;
3674	if (!pMixedCtx->ldtr.Attr.u)
3675	u32Access = HMVMX_SEL_UNUSABLE;
3676	else
3677	u32Access = pMixedCtx->ldtr.Attr.u;
3678
3679	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3680	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3681	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3682	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3683
3684	/* Validate. */
3685	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3686	{
3687	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3688	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3689	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3690	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3691	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3692	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3693	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3694	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3695	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3696	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3697	}
3698
3699	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3700	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3701	}
3702
3703	/*
3704	* Guest IDTR.
3705	*/
3706	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3707	{
3708	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3709	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3710
3711	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3712	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3713	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3714	}
3715
3716	return VINF_SUCCESS;
3717	}
3718
3719
3720	/**
3721	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3722	* areas. These MSRs will automatically be loaded to the host CPU on every
3723	* successful VM entry and stored from the host CPU on every successful VM exit.
3724	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3725	*
3726	* @returns VBox status code.
3727	* @param pVCpu Pointer to the VMCPU.
3728	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3729	* out-of-sync. Make sure to update the required fields
3730	* before using them.
3731	*
3732	* @remarks No-long-jump zone!!!
3733	*/
3734	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3735	{
3736	AssertPtr(pVCpu);
3737	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3738
3739	/*
3740	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3741	*/
3742	int rc = VINF_SUCCESS;
3743	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3744	{
3745	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3746	PVM pVM = pVCpu->CTX_SUFF(pVM);
3747	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3748	uint32_t cGuestMsrs = 0;
3749
3750	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3751	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3752	* when the guest really is in 64-bit mode. */
3753	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3754	if (fSupportsLongMode)
3755	{
3756	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3757	pGuestMsr->u32Reserved = 0;
3758	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3759	pGuestMsr++; cGuestMsrs++;
3760	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3761	pGuestMsr->u32Reserved = 0;
3762	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3763	pGuestMsr++; cGuestMsrs++;
3764	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3765	pGuestMsr->u32Reserved = 0;
3766	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3767	pGuestMsr++; cGuestMsrs++;
3768	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3769	pGuestMsr->u32Reserved = 0;
3770	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3771	pGuestMsr++; cGuestMsrs++;
3772	}
3773
3774	/*
3775	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3776	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3777	*/
3778	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3779	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3780	{
3781	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3782	pGuestMsr->u32Reserved = 0;
3783	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3784	AssertRCReturn(rc, rc);
3785	pGuestMsr++; cGuestMsrs++;
3786	}
3787
3788	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3789	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3790	{
3791	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3792	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3793	}
3794
3795	/* Update the VCPU's copy of the guest MSR count. */
3796	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3797	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3798	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3799	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3800
3801	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3802	}
3803
3804	/*
3805	* Guest Sysenter MSRs.
3806	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3807	* VM-exits on WRMSRs for these MSRs.
3808	*/
3809	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3810	{
3811	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3812	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3813	}
3814	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3815	{
3816	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3817	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3818	}
3819	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3820	{
3821	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3822	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3823	}
3824
3825	return rc;
3826	}
3827
3828
3829	/**
3830	* Loads the guest activity state into the guest-state area in the VMCS.
3831	*
3832	* @returns VBox status code.
3833	* @param pVCpu Pointer to the VMCPU.
3834	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3835	* out-of-sync. Make sure to update the required fields
3836	* before using them.
3837	*
3838	* @remarks No-long-jump zone!!!
3839	*/
3840	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3841	{
3842	/** @todo See if we can make use of other states, e.g.
3843	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3844	int rc = VINF_SUCCESS;
3845	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3846	{
3847	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3848	AssertRCReturn(rc, rc);
3849	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3850	}
3851	return rc;
3852	}
3853
3854
3855	/**
3856	* Sets up the appropriate function to run guest code.
3857	*
3858	* @returns VBox status code.
3859	* @param pVCpu Pointer to the VMCPU.
3860	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3861	* out-of-sync. Make sure to update the required fields
3862	* before using them.
3863	*
3864	* @remarks No-long-jump zone!!!
3865	*/
3866	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3867	{
3868	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3869	{
3870	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3871	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3872	#endif
3873	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3874	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3875	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3876	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3877	#else
3878	/* 64-bit host or hybrid host. */
3879	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3880	#endif
3881	}
3882	else
3883	{
3884	/* Guest is not in long mode, use the 32-bit handler. */
3885	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3886	}
3887	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3888	return VINF_SUCCESS;
3889	}
3890
3891
3892	/**
3893	* Wrapper for running the guest code in VT-x.
3894	*
3895	* @returns VBox strict status code.
3896	* @param pVM Pointer to the VM.
3897	* @param pVCpu Pointer to the VMCPU.
3898	* @param pCtx Pointer to the guest-CPU context.
3899	*
3900	* @remarks No-long-jump zone!!!
3901	*/
3902	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3903	{
3904	/*
3905	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3906	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3907	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3908	*/
3909	#ifdef VBOX_WITH_KERNEL_USING_XMM
3910	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3911	#else
3912	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3913	#endif
3914	}
3915
3916
3917	/**
3918	* Report world-switch error and dump some useful debug info.
3919	*
3920	* @param pVM Pointer to the VM.
3921	* @param pVCpu Pointer to the VMCPU.
3922	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3923	* @param pCtx Pointer to the guest-CPU context.
3924	* @param pVmxTransient Pointer to the VMX transient structure (only
3925	* exitReason updated).
3926	*/
3927	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3928	{
3929	Assert(pVM);
3930	Assert(pVCpu);
3931	Assert(pCtx);
3932	Assert(pVmxTransient);
3933	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3934
3935	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3936	switch (rcVMRun)
3937	{
3938	case VERR_VMX_INVALID_VMXON_PTR:
3939	AssertFailed();
3940	break;
3941	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3942	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3943	{
3944	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3945	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3946	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3947	AssertRC(rc);
3948
3949	#ifdef VBOX_STRICT
3950	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3951	pVmxTransient->uExitReason));
3952	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3953	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3954	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3955	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3956	else
3957	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3958
3959	/* VMX control bits. */
3960	uint32_t u32Val;
3961	uint64_t u64Val;
3962	HMVMXHCUINTREG uHCReg;
3963	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3964	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3965	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3966	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3967	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3968	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3969	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3970	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3971	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3972	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3973	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3974	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3975	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3976	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3977	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3978	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3979	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3980	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3981	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3982	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3983	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3984	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3985	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3986	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3987	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3988	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3989	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3990	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3991	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3992	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3993	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3994	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3995	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3996	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3997	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3998	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3999	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4000	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4001	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4002	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4003	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4004	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4005
4006	/* Guest bits. */
4007	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4008	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4009	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4010	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4011	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4012	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4013	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4014	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4015
4016	/* Host bits. */
4017	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4018	Log4(("Host CR0 %#RHr\n", uHCReg));
4019	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4020	Log4(("Host CR3 %#RHr\n", uHCReg));
4021	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4022	Log4(("Host CR4 %#RHr\n", uHCReg));
4023
4024	RTGDTR HostGdtr;
4025	PCX86DESCHC pDesc;
4026	ASMGetGDTR(&HostGdtr);
4027	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4028	Log4(("Host CS %#08x\n", u32Val));
4029	if (u32Val < HostGdtr.cbGdt)
4030	{
4031	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4032	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4033	}
4034
4035	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4036	Log4(("Host DS %#08x\n", u32Val));
4037	if (u32Val < HostGdtr.cbGdt)
4038	{
4039	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4040	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4041	}
4042
4043	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4044	Log4(("Host ES %#08x\n", u32Val));
4045	if (u32Val < HostGdtr.cbGdt)
4046	{
4047	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4048	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4049	}
4050
4051	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4052	Log4(("Host FS %#08x\n", u32Val));
4053	if (u32Val < HostGdtr.cbGdt)
4054	{
4055	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4056	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4057	}
4058
4059	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4060	Log4(("Host GS %#08x\n", u32Val));
4061	if (u32Val < HostGdtr.cbGdt)
4062	{
4063	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4064	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4065	}
4066
4067	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4068	Log4(("Host SS %#08x\n", u32Val));
4069	if (u32Val < HostGdtr.cbGdt)
4070	{
4071	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4072	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4073	}
4074
4075	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4076	Log4(("Host TR %#08x\n", u32Val));
4077	if (u32Val < HostGdtr.cbGdt)
4078	{
4079	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4080	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4081	}
4082
4083	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4084	Log4(("Host TR Base %#RHv\n", uHCReg));
4085	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4086	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4087	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4088	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4089	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4090	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4091	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4092	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4093	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4094	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4095	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4096	Log4(("Host RSP %#RHv\n", uHCReg));
4097	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4098	Log4(("Host RIP %#RHv\n", uHCReg));
4099	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4100	if (HMVMX_IS_64BIT_HOST_MODE())
4101	{
4102	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4103	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4104	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4105	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4106	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4107	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4108	}
4109	# endif
4110	#endif /* VBOX_STRICT */
4111	break;
4112	}
4113
4114	default:
4115	/* Impossible */
4116	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4117	break;
4118	}
4119	NOREF(pVM);
4120	}
4121
4122
4123	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4124	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4125	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4126	#endif
4127	#ifdef VBOX_STRICT
4128	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4129	{
4130	switch (idxField)
4131	{
4132	case VMX_VMCS_GUEST_RIP:
4133	case VMX_VMCS_GUEST_RSP:
4134	case VMX_VMCS_GUEST_SYSENTER_EIP:
4135	case VMX_VMCS_GUEST_SYSENTER_ESP:
4136	case VMX_VMCS_GUEST_GDTR_BASE:
4137	case VMX_VMCS_GUEST_IDTR_BASE:
4138	case VMX_VMCS_GUEST_CS_BASE:
4139	case VMX_VMCS_GUEST_DS_BASE:
4140	case VMX_VMCS_GUEST_ES_BASE:
4141	case VMX_VMCS_GUEST_FS_BASE:
4142	case VMX_VMCS_GUEST_GS_BASE:
4143	case VMX_VMCS_GUEST_SS_BASE:
4144	case VMX_VMCS_GUEST_LDTR_BASE:
4145	case VMX_VMCS_GUEST_TR_BASE:
4146	case VMX_VMCS_GUEST_CR3:
4147	return true;
4148	}
4149	return false;
4150	}
4151
4152	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4153	{
4154	switch (idxField)
4155	{
4156	/* Read-only fields. */
4157	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4158	return true;
4159	}
4160	/* Remaining readable fields should also be writable. */
4161	return hmR0VmxIsValidWriteField(idxField);
4162	}
4163	#endif /* VBOX_STRICT */
4164
4165
4166	/**
4167	* Executes the specified handler in 64-bit mode.
4168	*
4169	* @returns VBox status code.
4170	* @param pVM Pointer to the VM.
4171	* @param pVCpu Pointer to the VMCPU.
4172	* @param pCtx Pointer to the guest CPU context.
4173	* @param enmOp The operation to perform.
4174	* @param cbParam Number of parameters.
4175	* @param paParam Array of 32-bit parameters.
4176	*/
4177	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4178	uint32_t *paParam)
4179	{
4180	int rc, rc2;
4181	PHMGLOBLCPUINFO pCpu;
4182	RTHCPHYS HCPhysCpuPage;
4183	RTCCUINTREG uOldEFlags;
4184
4185	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4186	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4187	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4188	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4189
4190	#ifdef VBOX_STRICT
4191	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4192	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4193
4194	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4195	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4196	#endif
4197
4198	/* Disable interrupts. */
4199	uOldEFlags = ASMIntDisableFlags();
4200
4201	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4202	RTCPUID idHostCpu = RTMpCpuId();
4203	CPUMR0SetLApic(pVM, idHostCpu);
4204	#endif
4205
4206	pCpu = HMR0GetCurrentCpu();
4207	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4208
4209	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4210	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4211
4212	/* Leave VMX Root Mode. */
4213	VMXDisable();
4214
4215	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4216
4217	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4218	CPUMSetHyperEIP(pVCpu, enmOp);
4219	for (int i = (int)cbParam - 1; i >= 0; i--)
4220	CPUMPushHyper(pVCpu, paParam[i]);
4221
4222	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4223
4224	/* Call the switcher. */
4225	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4226	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4227
4228	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4229	/* Make sure the VMX instructions don't cause #UD faults. */
4230	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4231
4232	/* Re-enter VMX Root Mode */
4233	rc2 = VMXEnable(HCPhysCpuPage);
4234	if (RT_FAILURE(rc2))
4235	{
4236	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4237	ASMSetFlags(uOldEFlags);
4238	return rc2;
4239	}
4240
4241	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4242	AssertRC(rc2);
4243	Assert(!(ASMGetFlags() & X86_EFL_IF));
4244	ASMSetFlags(uOldEFlags);
4245	return rc;
4246	}
4247
4248
4249	/**
4250	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4251	* supporting 64-bit guests.
4252	*
4253	* @returns VBox status code.
4254	* @param fResume Whether to VMLAUNCH or VMRESUME.
4255	* @param pCtx Pointer to the guest-CPU context.
4256	* @param pCache Pointer to the VMCS cache.
4257	* @param pVM Pointer to the VM.
4258	* @param pVCpu Pointer to the VMCPU.
4259	*/
4260	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4261	{
4262	uint32_t aParam[6];
4263	PHMGLOBLCPUINFO pCpu = NULL;
4264	RTHCPHYS HCPhysCpuPage = 0;
4265	int rc = VERR_INTERNAL_ERROR_5;
4266
4267	pCpu = HMR0GetCurrentCpu();
4268	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4269
4270	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4271	pCache->uPos = 1;
4272	pCache->interPD = PGMGetInterPaeCR3(pVM);
4273	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4274	#endif
4275
4276	#ifdef VBOX_STRICT
4277	pCache->TestIn.HCPhysCpuPage = 0;
4278	pCache->TestIn.HCPhysVmcs = 0;
4279	pCache->TestIn.pCache = 0;
4280	pCache->TestOut.HCPhysVmcs = 0;
4281	pCache->TestOut.pCache = 0;
4282	pCache->TestOut.pCtx = 0;
4283	pCache->TestOut.eflags = 0;
4284	#endif
4285
4286	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4287	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4288	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4289	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4290	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4291	aParam[5] = 0;
4292
4293	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4294	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4295	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4296	#endif
4297	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4298
4299	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4300	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4301	Assert(pCtx->dr[4] == 10);
4302	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4303	#endif
4304
4305	#ifdef VBOX_STRICT
4306	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4307	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4308	pVCpu->hm.s.vmx.HCPhysVmcs));
4309	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4310	pCache->TestOut.HCPhysVmcs));
4311	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4312	pCache->TestOut.pCache));
4313	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4314	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4315	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4316	pCache->TestOut.pCtx));
4317	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4318	#endif
4319	return rc;
4320	}
4321
4322
4323	/**
4324	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4325	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4326	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4327	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4328	*
4329	* @returns VBox status code.
4330	* @param pVM Pointer to the VM.
4331	* @param pVCpu Pointer to the VMCPU.
4332	*/
4333	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4334	{
4335	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4336	{ \
4337	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4338	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4339	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4340	++cReadFields; \
4341	}
4342
4343	AssertPtr(pVM);
4344	AssertPtr(pVCpu);
4345	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4346	uint32_t cReadFields = 0;
4347
4348	/*
4349	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4350	* and serve to indicate exceptions to the rules.
4351	*/
4352
4353	/* Guest-natural selector base fields. */
4354	#if 0
4355	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4356	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4357	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4358	#endif
4359	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4360	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4361	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4362	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4363	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4364	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4365	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4366	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4367	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4368	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4369	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4370	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4371	#if 0
4372	/* Unused natural width guest-state fields. */
4373	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4374	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4375	#endif
4376	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4377	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4378
4379	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4380	#if 0
4381	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4382	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4383	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4384	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4385	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4387	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4388	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4390	#endif
4391
4392	/* Natural width guest-state fields. */
4393	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4394	#if 0
4395	/* Currently unused field. */
4396	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4397	#endif
4398
4399	if (pVM->hm.s.fNestedPaging)
4400	{
4401	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4402	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4403	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4404	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4405	}
4406	else
4407	{
4408	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4409	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4410	}
4411
4412	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4413	return VINF_SUCCESS;
4414	}
4415
4416
4417	/**
4418	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4419	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4420	* darwin, running 64-bit guests).
4421	*
4422	* @returns VBox status code.
4423	* @param pVCpu Pointer to the VMCPU.
4424	* @param idxField The VMCS field encoding.
4425	* @param u64Val 16, 32 or 64 bits value.
4426	*/
4427	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4428	{
4429	int rc;
4430	switch (idxField)
4431	{
4432	/*
4433	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4434	*/
4435	/* 64-bit Control fields. */
4436	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4437	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4438	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4439	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4440	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4441	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4442	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4443	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4444	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4445	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4446	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4447	case VMX_VMCS64_CTRL_EPTP_FULL:
4448	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4449	/* 64-bit Guest-state fields. */
4450	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4451	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4452	case VMX_VMCS64_GUEST_PAT_FULL:
4453	case VMX_VMCS64_GUEST_EFER_FULL:
4454	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4455	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4456	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4457	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4458	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4459	/* 64-bit Host-state fields. */
4460	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4461	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4462	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4463	{
4464	rc = VMXWriteVmcs32(idxField, u64Val);
4465	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4466	break;
4467	}
4468
4469	/*
4470	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4471	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4472	*/
4473	/* Natural-width Guest-state fields. */
4474	case VMX_VMCS_GUEST_CR3:
4475	case VMX_VMCS_GUEST_ES_BASE:
4476	case VMX_VMCS_GUEST_CS_BASE:
4477	case VMX_VMCS_GUEST_SS_BASE:
4478	case VMX_VMCS_GUEST_DS_BASE:
4479	case VMX_VMCS_GUEST_FS_BASE:
4480	case VMX_VMCS_GUEST_GS_BASE:
4481	case VMX_VMCS_GUEST_LDTR_BASE:
4482	case VMX_VMCS_GUEST_TR_BASE:
4483	case VMX_VMCS_GUEST_GDTR_BASE:
4484	case VMX_VMCS_GUEST_IDTR_BASE:
4485	case VMX_VMCS_GUEST_RSP:
4486	case VMX_VMCS_GUEST_RIP:
4487	case VMX_VMCS_GUEST_SYSENTER_ESP:
4488	case VMX_VMCS_GUEST_SYSENTER_EIP:
4489	{
4490	if (!(u64Val >> 32))
4491	{
4492	/* If this field is 64-bit, VT-x will zero out the top bits. */
4493	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4494	}
4495	else
4496	{
4497	/* Assert that only the 32->64 switcher case should ever come here. */
4498	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4499	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4500	}
4501	break;
4502	}
4503
4504	default:
4505	{
4506	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4507	rc = VERR_INVALID_PARAMETER;
4508	break;
4509	}
4510	}
4511	AssertRCReturn(rc, rc);
4512	return rc;
4513	}
4514
4515
4516	/**
4517	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4518	* hosts (except darwin) for 64-bit guests.
4519	*
4520	* @param pVCpu Pointer to the VMCPU.
4521	* @param idxField The VMCS field encoding.
4522	* @param u64Val 16, 32 or 64 bits value.
4523	*/
4524	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4525	{
4526	AssertPtr(pVCpu);
4527	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4528
4529	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4530	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4531
4532	/* Make sure there are no duplicates. */
4533	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4534	{
4535	if (pCache->Write.aField[i] == idxField)
4536	{
4537	pCache->Write.aFieldVal[i] = u64Val;
4538	return VINF_SUCCESS;
4539	}
4540	}
4541
4542	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4543	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4544	pCache->Write.cValidEntries++;
4545	return VINF_SUCCESS;
4546	}
4547
4548	/* Enable later when the assembly code uses these as callbacks. */
4549	#if 0
4550	/*
4551	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4552	*
4553	* @param pVCpu Pointer to the VMCPU.
4554	* @param pCache Pointer to the VMCS cache.
4555	*
4556	* @remarks No-long-jump zone!!!
4557	*/
4558	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4559	{
4560	AssertPtr(pCache);
4561	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4562	{
4563	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4564	AssertRC(rc);
4565	}
4566	pCache->Write.cValidEntries = 0;
4567	}
4568
4569
4570	/**
4571	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4572	*
4573	* @param pVCpu Pointer to the VMCPU.
4574	* @param pCache Pointer to the VMCS cache.
4575	*
4576	* @remarks No-long-jump zone!!!
4577	*/
4578	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4579	{
4580	AssertPtr(pCache);
4581	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4582	{
4583	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4584	AssertRC(rc);
4585	}
4586	}
4587	#endif
4588	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4589
4590
4591	/**
4592	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4593	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4594	* timer.
4595	*
4596	* @returns VBox status code.
4597	* @param pVCpu Pointer to the VMCPU.
4598	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4599	* out-of-sync. Make sure to update the required fields
4600	* before using them.
4601	* @remarks No-long-jump zone!!!
4602	*/
4603	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4604	{
4605	int rc = VERR_INTERNAL_ERROR_5;
4606	bool fOffsettedTsc = false;
4607	PVM pVM = pVCpu->CTX_SUFF(pVM);
4608	if (pVM->hm.s.vmx.fUsePreemptTimer)
4609	{
4610	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4611
4612	/* Make sure the returned values have sane upper and lower boundaries. */
4613	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4614	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4615	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4616	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4617
4618	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4619	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4620	}
4621	else
4622	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4623
4624	if (fOffsettedTsc)
4625	{
4626	uint64_t u64CurTSC = ASMReadTSC();
4627	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4628	{
4629	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4630	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4631
4632	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4633	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4634	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4635	}
4636	else
4637	{
4638	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4639	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4640	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4641	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4642	}
4643	}
4644	else
4645	{
4646	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4647	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4648	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4649	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4650	}
4651	}
4652
4653
4654	/**
4655	* Determines if an exception is a contributory exception. Contributory
4656	* exceptions are ones which can cause double-faults. Page-fault is
4657	* intentionally not included here as it's a conditional contributory exception.
4658	*
4659	* @returns true if the exception is contributory, false otherwise.
4660	* @param uVector The exception vector.
4661	*/
4662	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4663	{
4664	switch (uVector)
4665	{
4666	case X86_XCPT_GP:
4667	case X86_XCPT_SS:
4668	case X86_XCPT_NP:
4669	case X86_XCPT_TS:
4670	case X86_XCPT_DE:
4671	return true;
4672	default:
4673	break;
4674	}
4675	return false;
4676	}
4677
4678
4679	/**
4680	* Sets an event as a pending event to be injected into the guest.
4681	*
4682	* @param pVCpu Pointer to the VMCPU.
4683	* @param u32IntrInfo The VM-entry interruption-information field.
4684	* @param cbInstr The VM-entry instruction length in bytes (for software
4685	* interrupts, exceptions and privileged software
4686	* exceptions).
4687	* @param u32ErrCode The VM-entry exception error code.
4688	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4689	* page-fault.
4690	*/
4691	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4692	RTGCUINTPTR GCPtrFaultAddress)
4693	{
4694	Assert(!pVCpu->hm.s.Event.fPending);
4695	pVCpu->hm.s.Event.fPending = true;
4696	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4697	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4698	pVCpu->hm.s.Event.cbInstr = cbInstr;
4699	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4700	}
4701
4702
4703	/**
4704	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4705	*
4706	* @param pVCpu Pointer to the VMCPU.
4707	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4708	* out-of-sync. Make sure to update the required fields
4709	* before using them.
4710	*/
4711	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4712	{
4713	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4714	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4715	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4716	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4717	}
4718
4719
4720	/**
4721	* Handle a condition that occurred while delivering an event through the guest
4722	* IDT.
4723	*
4724	* @returns VBox status code (informational error codes included).
4725	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4726	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4727	* continue execution of the guest which will delivery the #DF.
4728	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4729	*
4730	* @param pVCpu Pointer to the VMCPU.
4731	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4732	* out-of-sync. Make sure to update the required fields
4733	* before using them.
4734	* @param pVmxTransient Pointer to the VMX transient structure.
4735	*
4736	* @remarks No-long-jump zone!!!
4737	*/
4738	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4739	{
4740	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4741	AssertRC(rc);
4742	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4743	{
4744	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4745	AssertRCReturn(rc, rc);
4746
4747	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4748	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4749	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4750
4751	typedef enum
4752	{
4753	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4754	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4755	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4756	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4757	} VMXREFLECTXCPT;
4758
4759	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4760	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4761	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4762	{
4763	enmReflect = VMXREFLECTXCPT_XCPT;
4764	#ifdef VBOX_STRICT
4765	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4766	&& uExitVector == X86_XCPT_PF)
4767	{
4768	Log4(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4769	}
4770	#endif
4771	if ( uExitVector == X86_XCPT_PF
4772	&& uIdtVector == X86_XCPT_PF)
4773	{
4774	pVmxTransient->fVectoringPF = true;
4775	Log4(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4776	}
4777	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4778	&& hmR0VmxIsContributoryXcpt(uExitVector)
4779	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4780	\|\| uIdtVector == X86_XCPT_PF))
4781	{
4782	enmReflect = VMXREFLECTXCPT_DF;
4783	}
4784	else if (uIdtVector == X86_XCPT_DF)
4785	enmReflect = VMXREFLECTXCPT_TF;
4786	}
4787	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4788	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4789	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4790	{
4791	/*
4792	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4793	* (whatever they are) as they reoccur when restarting the instruction.
4794	*/
4795	enmReflect = VMXREFLECTXCPT_XCPT;
4796	}
4797
4798	switch (enmReflect)
4799	{
4800	case VMXREFLECTXCPT_XCPT:
4801	{
4802	uint32_t u32ErrCode = 0;
4803	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4804	{
4805	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4806	AssertRCReturn(rc, rc);
4807	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4808	}
4809
4810	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4811	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4812	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4813	rc = VINF_SUCCESS;
4814	Log4(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4815	pVCpu->hm.s.Event.u32ErrCode));
4816	break;
4817	}
4818
4819	case VMXREFLECTXCPT_DF:
4820	{
4821	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4822	rc = VINF_VMX_DOUBLE_FAULT;
4823	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4824	uIdtVector, uExitVector));
4825	break;
4826	}
4827
4828	case VMXREFLECTXCPT_TF:
4829	{
4830	Log4(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4831	rc = VINF_EM_RESET;
4832	break;
4833	}
4834
4835	default:
4836	Assert(rc == VINF_SUCCESS);
4837	break;
4838	}
4839	}
4840	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4841	return rc;
4842	}
4843
4844
4845	/**
4846	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4847	*
4848	* @returns VBox status code.
4849	* @param pVCpu Pointer to the VMCPU.
4850	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4851	* out-of-sync. Make sure to update the required fields
4852	* before using them.
4853	*
4854	* @remarks No-long-jump zone!!!
4855	*/
4856	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4857	{
4858	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4859	{
4860	uint32_t uVal = 0;
4861	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4862	AssertRCReturn(rc, rc);
4863	uint32_t uShadow = 0;
4864	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4865	AssertRCReturn(rc, rc);
4866
4867	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4868	CPUMSetGuestCR0(pVCpu, uVal);
4869	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4870	}
4871	return VINF_SUCCESS;
4872	}
4873
4874
4875	/**
4876	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4877	*
4878	* @returns VBox status code.
4879	* @param pVCpu Pointer to the VMCPU.
4880	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4881	* out-of-sync. Make sure to update the required fields
4882	* before using them.
4883	*
4884	* @remarks No-long-jump zone!!!
4885	*/
4886	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4887	{
4888	int rc = VINF_SUCCESS;
4889	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4890	{
4891	uint32_t uVal = 0;
4892	uint32_t uShadow = 0;
4893	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4894	AssertRCReturn(rc, rc);
4895	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4896	AssertRCReturn(rc, rc);
4897
4898	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4899	CPUMSetGuestCR4(pVCpu, uVal);
4900	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4901	}
4902	return rc;
4903	}
4904
4905
4906	/**
4907	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4908	*
4909	* @returns VBox status code.
4910	* @param pVCpu Pointer to the VMCPU.
4911	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4912	* out-of-sync. Make sure to update the required fields
4913	* before using them.
4914	*
4915	* @remarks No-long-jump zone!!!
4916	*/
4917	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4918	{
4919	int rc = VINF_SUCCESS;
4920	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4921	{
4922	uint64_t u64Val = 0;
4923	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4924	AssertRCReturn(rc, rc);
4925
4926	pMixedCtx->rip = u64Val;
4927	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4928	}
4929	return rc;
4930	}
4931
4932
4933	/**
4934	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4935	*
4936	* @returns VBox status code.
4937	* @param pVCpu Pointer to the VMCPU.
4938	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4939	* out-of-sync. Make sure to update the required fields
4940	* before using them.
4941	*
4942	* @remarks No-long-jump zone!!!
4943	*/
4944	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4945	{
4946	int rc = VINF_SUCCESS;
4947	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4948	{
4949	uint64_t u64Val = 0;
4950	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4951	AssertRCReturn(rc, rc);
4952
4953	pMixedCtx->rsp = u64Val;
4954	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4955	}
4956	return rc;
4957	}
4958
4959
4960	/**
4961	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4962	*
4963	* @returns VBox status code.
4964	* @param pVCpu Pointer to the VMCPU.
4965	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4966	* out-of-sync. Make sure to update the required fields
4967	* before using them.
4968	*
4969	* @remarks No-long-jump zone!!!
4970	*/
4971	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4972	{
4973	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4974	{
4975	uint32_t uVal = 0;
4976	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4977	AssertRCReturn(rc, rc);
4978
4979	pMixedCtx->eflags.u32 = uVal;
4980	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4981	{
4982	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4983	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4984
4985	pMixedCtx->eflags.Bits.u1VM = 0;
4986	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4987	}
4988
4989	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4990	}
4991	return VINF_SUCCESS;
4992	}
4993
4994
4995	/**
4996	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4997	* guest-CPU context.
4998	*/
4999	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5000	{
5001	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5002	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5003	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5004	return rc;
5005	}
5006
5007
5008	/**
5009	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5010	* from the guest-state area in the VMCS.
5011	*
5012	* @param pVCpu Pointer to the VMCPU.
5013	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5014	* out-of-sync. Make sure to update the required fields
5015	* before using them.
5016	*
5017	* @remarks No-long-jump zone!!!
5018	*/
5019	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5020	{
5021	uint32_t uIntrState = 0;
5022	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5023	AssertRC(rc);
5024
5025	if (!uIntrState)
5026	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5027	else
5028	{
5029	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5030	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5031	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5032	AssertRC(rc);
5033	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5034	AssertRC(rc);
5035
5036	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5037	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5038	}
5039	}
5040
5041
5042	/**
5043	* Saves the guest's activity state.
5044	*
5045	* @returns VBox status code.
5046	* @param pVCpu Pointer to the VMCPU.
5047	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5048	* out-of-sync. Make sure to update the required fields
5049	* before using them.
5050	*
5051	* @remarks No-long-jump zone!!!
5052	*/
5053	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5054	{
5055	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5056	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5057	return VINF_SUCCESS;
5058	}
5059
5060
5061	/**
5062	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5063	* the current VMCS into the guest-CPU context.
5064	*
5065	* @returns VBox status code.
5066	* @param pVCpu Pointer to the VMCPU.
5067	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5068	* out-of-sync. Make sure to update the required fields
5069	* before using them.
5070	*
5071	* @remarks No-long-jump zone!!!
5072	*/
5073	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5074	{
5075	int rc = VINF_SUCCESS;
5076	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5077	{
5078	uint32_t u32Val = 0;
5079	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5080	pMixedCtx->SysEnter.cs = u32Val;
5081	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5082	}
5083
5084	uint64_t u64Val = 0;
5085	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5086	{
5087	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5088	pMixedCtx->SysEnter.eip = u64Val;
5089	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5090	}
5091	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5092	{
5093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5094	pMixedCtx->SysEnter.esp = u64Val;
5095	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5096	}
5097	return rc;
5098	}
5099
5100
5101	/**
5102	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5103	* context.
5104	*
5105	* @returns VBox status code.
5106	* @param pVCpu Pointer to the VMCPU.
5107	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5108	* out-of-sync. Make sure to update the required fields
5109	* before using them.
5110	*
5111	* @remarks No-long-jump zone!!!
5112	*/
5113	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5114	{
5115	int rc = VINF_SUCCESS;
5116	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5117	{
5118	uint64_t u64Val = 0;
5119	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5120	pMixedCtx->fs.u64Base = u64Val;
5121	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5122	}
5123	return rc;
5124	}
5125
5126
5127	/**
5128	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5129	* context.
5130	*
5131	* @returns VBox status code.
5132	* @param pVCpu Pointer to the VMCPU.
5133	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5134	* out-of-sync. Make sure to update the required fields
5135	* before using them.
5136	*
5137	* @remarks No-long-jump zone!!!
5138	*/
5139	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5140	{
5141	int rc = VINF_SUCCESS;
5142	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5143	{
5144	uint64_t u64Val = 0;
5145	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5146	pMixedCtx->gs.u64Base = u64Val;
5147	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5148	}
5149	return rc;
5150	}
5151
5152
5153	/**
5154	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5155	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5156	* and TSC_AUX.
5157	*
5158	* @returns VBox status code.
5159	* @param pVCpu Pointer to the VMCPU.
5160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5161	* out-of-sync. Make sure to update the required fields
5162	* before using them.
5163	*
5164	* @remarks No-long-jump zone!!!
5165	*/
5166	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5167	{
5168	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5169	return VINF_SUCCESS;
5170
5171	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5172	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5173	{
5174	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5175	pMsr += i;
5176	switch (pMsr->u32IndexMSR)
5177	{
5178	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5179	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5180	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5181	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5182	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5183	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5184	default:
5185	{
5186	AssertFailed();
5187	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5188	}
5189	}
5190	}
5191	#endif
5192
5193	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5194	return VINF_SUCCESS;
5195	}
5196
5197
5198	/**
5199	* Saves the guest control registers from the current VMCS into the guest-CPU
5200	* context.
5201	*
5202	* @returns VBox status code.
5203	* @param pVCpu Pointer to the VMCPU.
5204	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5205	* out-of-sync. Make sure to update the required fields
5206	* before using them.
5207	*
5208	* @remarks No-long-jump zone!!!
5209	*/
5210	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5211	{
5212	/* Guest CR0. Guest FPU. */
5213	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5214	AssertRCReturn(rc, rc);
5215
5216	/* Guest CR4. */
5217	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5218	AssertRCReturn(rc, rc);
5219
5220	/* Guest CR2 - updated always during the world-switch or in #PF. */
5221	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5222	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5223	{
5224	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5225	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5226
5227	PVM pVM = pVCpu->CTX_SUFF(pVM);
5228	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5229	\|\| ( pVM->hm.s.fNestedPaging
5230	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5231	{
5232	uint64_t u64Val = 0;
5233	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5234	if (pMixedCtx->cr3 != u64Val)
5235	{
5236	CPUMSetGuestCR3(pVCpu, u64Val);
5237	if (VMMRZCallRing3IsEnabled(pVCpu))
5238	{
5239	PGMUpdateCR3(pVCpu, u64Val);
5240	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5241	}
5242	else
5243	{
5244	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5245	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5246	}
5247	}
5248
5249	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5250	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5251	{
5252	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5253	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5254	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5255	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5256
5257	if (VMMRZCallRing3IsEnabled(pVCpu))
5258	{
5259	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5260	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5261	}
5262	else
5263	{
5264	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5265	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5266	}
5267	}
5268	}
5269
5270	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5271	}
5272
5273	/*
5274	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5275	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5276	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5277	*
5278	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5279	*/
5280	if (VMMRZCallRing3IsEnabled(pVCpu))
5281	{
5282	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5283	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5284
5285	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5286	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5287
5288	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5289	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5290	}
5291
5292	return rc;
5293	}
5294
5295
5296	/**
5297	* Reads a guest segment register from the current VMCS into the guest-CPU
5298	* context.
5299	*
5300	* @returns VBox status code.
5301	* @param pVCpu Pointer to the VMCPU.
5302	* @param idxSel Index of the selector in the VMCS.
5303	* @param idxLimit Index of the segment limit in the VMCS.
5304	* @param idxBase Index of the segment base in the VMCS.
5305	* @param idxAccess Index of the access rights of the segment in the VMCS.
5306	* @param pSelReg Pointer to the segment selector.
5307	*
5308	* @remarks No-long-jump zone!!!
5309	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5310	* macro as that takes care of whether to read from the VMCS cache or
5311	* not.
5312	*/
5313	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5314	PCPUMSELREG pSelReg)
5315	{
5316	uint32_t u32Val = 0;
5317	int rc = VMXReadVmcs32(idxSel, &u32Val);
5318	AssertRCReturn(rc, rc);
5319	pSelReg->Sel = (uint16_t)u32Val;
5320	pSelReg->ValidSel = (uint16_t)u32Val;
5321	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5322
5323	rc = VMXReadVmcs32(idxLimit, &u32Val);
5324	AssertRCReturn(rc, rc);
5325	pSelReg->u32Limit = u32Val;
5326
5327	uint64_t u64Val = 0;
5328	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5329	AssertRCReturn(rc, rc);
5330	pSelReg->u64Base = u64Val;
5331
5332	rc = VMXReadVmcs32(idxAccess, &u32Val);
5333	AssertRCReturn(rc, rc);
5334	pSelReg->Attr.u = u32Val;
5335
5336	/*
5337	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5338	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5339	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5340	*
5341	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5342	*/
5343	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5344	{
5345	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5346	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5347	}
5348	return VINF_SUCCESS;
5349	}
5350
5351
5352	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5353	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5354	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5355	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5356	#else
5357	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5358	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5359	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5360	#endif
5361
5362
5363	/**
5364	* Saves the guest segment registers from the current VMCS into the guest-CPU
5365	* context.
5366	*
5367	* @returns VBox status code.
5368	* @param pVCpu Pointer to the VMCPU.
5369	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5370	* out-of-sync. Make sure to update the required fields
5371	* before using them.
5372	*
5373	* @remarks No-long-jump zone!!!
5374	*/
5375	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376	{
5377	/* Guest segment registers. */
5378	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5379	{
5380	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5381	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5382	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5383	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5384	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5385	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5386	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5387
5388	/* Restore segment attributes for real-on-v86 mode hack. */
5389	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5390	{
5391	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5392	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5393	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5394	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5395	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5396	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5397	}
5398	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5399	}
5400
5401	return VINF_SUCCESS;
5402	}
5403
5404
5405	/**
5406	* Saves the guest descriptor table registers and task register from the current
5407	* VMCS into the guest-CPU context.
5408	*
5409	* @returns VBox status code.
5410	* @param pVCpu Pointer to the VMCPU.
5411	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5412	* out-of-sync. Make sure to update the required fields
5413	* before using them.
5414	*
5415	* @remarks No-long-jump zone!!!
5416	*/
5417	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5418	{
5419	int rc = VINF_SUCCESS;
5420
5421	/* Guest LDTR. */
5422	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5423	{
5424	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5425	AssertRCReturn(rc, rc);
5426	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5427	}
5428
5429	/* Guest GDTR. */
5430	uint64_t u64Val = 0;
5431	uint32_t u32Val = 0;
5432	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5433	{
5434	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5435	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5436	pMixedCtx->gdtr.pGdt = u64Val;
5437	pMixedCtx->gdtr.cbGdt = u32Val;
5438	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5439	}
5440
5441	/* Guest IDTR. */
5442	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5443	{
5444	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5445	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5446	pMixedCtx->idtr.pIdt = u64Val;
5447	pMixedCtx->idtr.cbIdt = u32Val;
5448	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5449	}
5450
5451	/* Guest TR. */
5452	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5453	{
5454	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5455	AssertRCReturn(rc, rc);
5456
5457	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5458	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5459	{
5460	rc = VMXLOCAL_READ_SEG(TR, tr);
5461	AssertRCReturn(rc, rc);
5462	}
5463	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5464	}
5465	return rc;
5466	}
5467
5468	#undef VMXLOCAL_READ_SEG
5469
5470
5471	/**
5472	* Saves the guest debug registers from the current VMCS into the guest-CPU
5473	* context.
5474	*
5475	* @returns VBox status code.
5476	* @param pVCpu Pointer to the VMCPU.
5477	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5478	* out-of-sync. Make sure to update the required fields
5479	* before using them.
5480	*
5481	* @remarks No-long-jump zone!!!
5482	*/
5483	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5484	{
5485	int rc = VINF_SUCCESS;
5486	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5487	{
5488	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5489	uint32_t u32Val;
5490	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5491	pMixedCtx->dr[7] = u32Val;
5492
5493	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5494	}
5495	return rc;
5496	}
5497
5498
5499	/**
5500	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5501	*
5502	* @returns VBox status code.
5503	* @param pVCpu Pointer to the VMCPU.
5504	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5505	* out-of-sync. Make sure to update the required fields
5506	* before using them.
5507	*
5508	* @remarks No-long-jump zone!!!
5509	*/
5510	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5511	{
5512	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5513	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5514	return VINF_SUCCESS;
5515	}
5516
5517
5518	/**
5519	* Saves the entire guest state from the currently active VMCS into the
5520	* guest-CPU context. This essentially VMREADs all guest-data.
5521	*
5522	* @returns VBox status code.
5523	* @param pVCpu Pointer to the VMCPU.
5524	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5525	* out-of-sync. Make sure to update the required fields
5526	* before using them.
5527	*/
5528	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5529	{
5530	Assert(pVCpu);
5531	Assert(pMixedCtx);
5532
5533	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5534	return VINF_SUCCESS;
5535
5536	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5537	there is no real need to. */
5538	if (VMMRZCallRing3IsEnabled(pVCpu))
5539	VMMR0LogFlushDisable(pVCpu);
5540	else
5541	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5542	Log4Func(("\n"));
5543
5544	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5545	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5546
5547	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5548	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5549
5550	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5551	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5552
5553	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5554	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5555
5556	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5557	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5558
5559	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5560	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5561
5562	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5563	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5564
5565	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5566	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5567
5568	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5569	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5570
5571	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5572	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5573
5574	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5575	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5576
5577	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5578	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5579
5580	if (VMMRZCallRing3IsEnabled(pVCpu))
5581	VMMR0LogFlushEnable(pVCpu);
5582
5583	return rc;
5584	}
5585
5586
5587	/**
5588	* Check per-VM and per-VCPU force flag actions that require us to go back to
5589	* ring-3 for one reason or another.
5590	*
5591	* @returns VBox status code (information status code included).
5592	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5593	* ring-3.
5594	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5595	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5596	* interrupts)
5597	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5598	* all EMTs to be in ring-3.
5599	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5600	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5601	* to the EM loop.
5602	*
5603	* @param pVM Pointer to the VM.
5604	* @param pVCpu Pointer to the VMCPU.
5605	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5606	* out-of-sync. Make sure to update the required fields
5607	* before using them.
5608	*/
5609	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5610	{
5611	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5612
5613	int rc = VERR_INTERNAL_ERROR_5;
5614	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5615	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5616	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5617	{
5618	/* We need the control registers now, make sure the guest-CPU context is updated. */
5619	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5620	AssertRCReturn(rc, rc);
5621
5622	/* Pending HM CR3 sync. */
5623	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5624	{
5625	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5626	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5627	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5628	}
5629
5630	/* Pending HM PAE PDPEs. */
5631	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5632	{
5633	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5634	AssertRC(rc);
5635	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5636	}
5637
5638	/* Pending PGM C3 sync. */
5639	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5640	{
5641	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5642	if (rc != VINF_SUCCESS)
5643	{
5644	AssertRC(rc);
5645	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5646	return rc;
5647	}
5648	}
5649
5650	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5651	/* -XXX- what was that about single stepping? */
5652	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5653	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5654	{
5655	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5656	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5657	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5658	return rc;
5659	}
5660
5661	/* Pending VM request packets, such as hardware interrupts. */
5662	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5663	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5664	{
5665	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5666	return VINF_EM_PENDING_REQUEST;
5667	}
5668
5669	/* Pending PGM pool flushes. */
5670	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5671	{
5672	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5673	return VINF_PGM_POOL_FLUSH_PENDING;
5674	}
5675
5676	/* Pending DMA requests. */
5677	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5678	{
5679	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5680	return VINF_EM_RAW_TO_R3;
5681	}
5682	}
5683
5684	/* Paranoia. */
5685	Assert(rc != VERR_EM_INTERPRETER);
5686	return VINF_SUCCESS;
5687	}
5688
5689
5690	/**
5691	* Converts any TRPM trap into a pending VMX event. This is typically used when
5692	* entering from ring-3 (not longjmp returns).
5693	*
5694	* @param pVCpu Pointer to the VMCPU.
5695	*/
5696	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5697	{
5698	Assert(TRPMHasTrap(pVCpu));
5699	Assert(!pVCpu->hm.s.Event.fPending);
5700
5701	uint8_t uVector;
5702	TRPMEVENT enmTrpmEvent;
5703	RTGCUINT uErrCode;
5704	RTGCUINTPTR GCPtrFaultAddress;
5705	uint8_t cbInstr;
5706
5707	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5708	AssertRC(rc);
5709
5710	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5711	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5712	if (enmTrpmEvent == TRPM_TRAP)
5713	{
5714	switch (uVector)
5715	{
5716	case X86_XCPT_BP:
5717	case X86_XCPT_OF:
5718	{
5719	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5720	break;
5721	}
5722
5723	case X86_XCPT_PF:
5724	case X86_XCPT_DF:
5725	case X86_XCPT_TS:
5726	case X86_XCPT_NP:
5727	case X86_XCPT_SS:
5728	case X86_XCPT_GP:
5729	case X86_XCPT_AC:
5730	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5731	/* no break! */
5732	default:
5733	{
5734	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5735	break;
5736	}
5737	}
5738	}
5739	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5740	{
5741	if (uVector != X86_XCPT_NMI)
5742	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5743	else
5744	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5745	}
5746	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5747	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5748	else
5749	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5750
5751	rc = TRPMResetTrap(pVCpu);
5752	AssertRC(rc);
5753	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5754	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5755	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5756	}
5757
5758
5759	/**
5760	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5761	* VT-x to execute any instruction.
5762	*
5763	* @param pvCpu Pointer to the VMCPU.
5764	*/
5765	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5766	{
5767	Assert(pVCpu->hm.s.Event.fPending);
5768
5769	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5770	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5771	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5772	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5773
5774	/* If a trap was already pending, we did something wrong! */
5775	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5776
5777	TRPMEVENT enmTrapType;
5778	switch (uVectorType)
5779	{
5780	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5781	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5782	enmTrapType = TRPM_HARDWARE_INT;
5783	break;
5784	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5785	enmTrapType = TRPM_SOFTWARE_INT;
5786	break;
5787	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5788	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5789	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5790	enmTrapType = TRPM_TRAP;
5791	break;
5792	default:
5793	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5794	enmTrapType = TRPM_32BIT_HACK;
5795	break;
5796	}
5797
5798	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5799
5800	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5801	AssertRC(rc);
5802
5803	if (fErrorCodeValid)
5804	TRPMSetErrorCode(pVCpu, uErrorCode);
5805	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5806	&& uVector == X86_XCPT_PF)
5807	{
5808	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5809	}
5810	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5811	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5812	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5813	{
5814	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5815	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5816	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5817	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5818	}
5819	pVCpu->hm.s.Event.fPending = false;
5820	}
5821
5822
5823	/**
5824	* Does the necessary state syncing before doing a longjmp to ring-3.
5825	*
5826	* @param pVM Pointer to the VM.
5827	* @param pVCpu Pointer to the VMCPU.
5828	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5829	* out-of-sync. Make sure to update the required fields
5830	* before using them.
5831	* @param rcExit The reason for exiting to ring-3. Can be
5832	* VINF_VMM_UNKNOWN_RING3_CALL.
5833	*
5834	* @remarks No-long-jmp zone!!!
5835	*/
5836	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5837	{
5838	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5839	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5840
5841	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5842	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5843	AssertRC(rc);
5844
5845	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5846	if (CPUMIsGuestFPUStateActive(pVCpu))
5847	{
5848	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5849	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5850	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5851	}
5852
5853	/* Restore debug registers if necessary and resync on next R0 reentry. */
5854	if (CPUMIsGuestDebugStateActive(pVCpu))
5855	{
5856	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5857	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5858	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5859	}
5860	else if (CPUMIsHyperDebugStateActive(pVCpu))
5861	{
5862	CPUMR0LoadHostDebugState(pVM, pVCpu);
5863	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5864	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5865	}
5866
5867	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5868	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
5869	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5870	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5871	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5872	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5873	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5874	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5875	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5876	}
5877
5878
5879	/**
5880	* An action requires us to go back to ring-3. This function does the necessary
5881	* steps before we can safely return to ring-3. This is not the same as longjmps
5882	* to ring-3, this is voluntary.
5883	*
5884	* @param pVM Pointer to the VM.
5885	* @param pVCpu Pointer to the VMCPU.
5886	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5887	* out-of-sync. Make sure to update the required fields
5888	* before using them.
5889	* @param rcExit The reason for exiting to ring-3. Can be
5890	* VINF_VMM_UNKNOWN_RING3_CALL.
5891	*/
5892	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5893	{
5894	Assert(pVM);
5895	Assert(pVCpu);
5896	Assert(pMixedCtx);
5897	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5898
5899	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5900	{
5901	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5902	return;
5903	}
5904	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5905	{
5906	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5907	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5908	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5909	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5910	return;
5911	}
5912
5913	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5914	VMMRZCallRing3Disable(pVCpu);
5915	Log4(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5916
5917	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5918	if (pVCpu->hm.s.Event.fPending)
5919	{
5920	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5921	Assert(!pVCpu->hm.s.Event.fPending);
5922	}
5923
5924	/* Sync. the guest state. */
5925	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5926	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5927
5928	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5929	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5930	\| CPUM_CHANGED_LDTR
5931	\| CPUM_CHANGED_GDTR
5932	\| CPUM_CHANGED_IDTR
5933	\| CPUM_CHANGED_TR
5934	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5935
5936	/* On our way back from ring-3 the following needs to be done. */
5937	/** @todo This can change with preemption hooks. */
5938	if (rcExit == VINF_EM_RAW_INTERRUPT)
5939	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5940	else
5941	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5942
5943	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5944	VMMRZCallRing3Enable(pVCpu);
5945	}
5946
5947
5948	/**
5949	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5950	* longjump to ring-3 and possibly get preempted.
5951	*
5952	* @param pVCpu Pointer to the VMCPU.
5953	* @param enmOperation The operation causing the ring-3 longjump.
5954	* @param pvUser The user argument (pointer to the possibly
5955	* out-of-date guest-CPU context).
5956	*
5957	* @remarks Must never be called with @a enmOperation ==
5958	* VMMCALLRING3_VM_R0_ASSERTION.
5959	*/
5960	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5961	{
5962	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5963	Assert(pVCpu);
5964	Assert(pvUser);
5965	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5966	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5967
5968	VMMRZCallRing3Disable(pVCpu);
5969	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5970	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5971	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5972	VMMRZCallRing3Enable(pVCpu);
5973	}
5974
5975
5976	/**
5977	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5978	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5979	*
5980	* @returns VBox status code.
5981	* @param pVCpu Pointer to the VMCPU.
5982	*/
5983	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5984	{
5985	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5986	{
5987	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5988	{
5989	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
5990	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5991	AssertRC(rc);
5992	}
5993	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5994	}
5995
5996
5997	/**
5998	* Injects any pending events into the guest if the guest is in a state to
5999	* receive them.
6000	*
6001	* @returns VBox status code (informational status codes included).
6002	* @param pVCpu Pointer to the VMCPU.
6003	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6004	* out-of-sync. Make sure to update the required fields
6005	* before using them.
6006	*/
6007	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6008	{
6009	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6010	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6011	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6012	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6013
6014	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6015	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6016	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6017	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6018	Assert(!TRPMHasTrap(pVCpu));
6019
6020	int rc = VINF_SUCCESS;
6021	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6022	{
6023	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6024	bool fInject = true;
6025	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6026	{
6027	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6028	AssertRCReturn(rc, rc);
6029	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6030	if ( fBlockInt
6031	\|\| fBlockSti
6032	\|\| fBlockMovSS)
6033	{
6034	fInject = false;
6035	}
6036	}
6037	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6038	&& ( fBlockMovSS
6039	\|\| fBlockSti))
6040	{
6041	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6042	fInject = false;
6043	}
6044
6045	if (fInject)
6046	{
6047	Log4(("Injecting pending event\n"));
6048	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6049	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6050	AssertRCReturn(rc, rc);
6051	pVCpu->hm.s.Event.fPending = false;
6052	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
6053	}
6054	else
6055	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6056	} /** @todo SMI. SMIs take priority over NMIs. */
6057	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6058	{
6059	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6060	if ( !fBlockMovSS
6061	&& !fBlockSti)
6062	{
6063	Log4(("Injecting NMI\n"));
6064	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6065	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6066	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6067	0 /* GCPtrFaultAddress */, &uIntrState);
6068	AssertRCReturn(rc, rc);
6069	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6070	}
6071	else
6072	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6073	}
6074	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6075	{
6076	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6077	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6078	AssertRCReturn(rc, rc);
6079	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6080	if ( !fBlockInt
6081	&& !fBlockSti
6082	&& !fBlockMovSS)
6083	{
6084	uint8_t u8Interrupt;
6085	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6086	if (RT_SUCCESS(rc))
6087	{
6088	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6089	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6090	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6091	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6092	0 /* GCPtrFaultAddress */, &uIntrState);
6093	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6094	}
6095	else
6096	{
6097	/** @todo Does this actually happen? If not turn it into an assertion. */
6098	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6099	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6100	rc = VINF_SUCCESS;
6101	}
6102	}
6103	else
6104	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6105	}
6106
6107	/*
6108	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6109	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6110	*/
6111	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6112	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6113	int rc2 = VINF_SUCCESS;
6114	if ( fBlockSti
6115	\|\| fBlockMovSS)
6116	{
6117	if (!DBGFIsStepping(pVCpu))
6118	{
6119	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6120	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6121	{
6122	/*
6123	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6124	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6125	*/
6126	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6127	AssertRCReturn(rc, rc);
6128	}
6129	}
6130	else
6131	{
6132	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6133	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6134	uIntrState = 0;
6135	}
6136	}
6137
6138	/*
6139	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6140	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6141	*/
6142	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6143	AssertRC(rc2);
6144
6145	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6146	return rc;
6147	}
6148
6149
6150	/**
6151	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6152	*
6153	* @param pVCpu Pointer to the VMCPU.
6154	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6155	* out-of-sync. Make sure to update the required fields
6156	* before using them.
6157	*/
6158	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6159	{
6160	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6161	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6162	}
6163
6164
6165	/**
6166	* Injects a double-fault (#DF) exception into the VM.
6167	*
6168	* @returns VBox status code (informational status code included).
6169	* @param pVCpu Pointer to the VMCPU.
6170	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6171	* out-of-sync. Make sure to update the required fields
6172	* before using them.
6173	*/
6174	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6175	{
6176	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6177	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6178	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6179	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6180	puIntrState);
6181	}
6182
6183
6184	/**
6185	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6186	*
6187	* @param pVCpu Pointer to the VMCPU.
6188	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6189	* out-of-sync. Make sure to update the required fields
6190	* before using them.
6191	*/
6192	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6193	{
6194	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6195	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6196	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6197	}
6198
6199
6200	/**
6201	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6202	*
6203	* @param pVCpu Pointer to the VMCPU.
6204	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6205	* out-of-sync. Make sure to update the required fields
6206	* before using them.
6207	* @param cbInstr The value of RIP that is to be pushed on the guest
6208	* stack.
6209	*/
6210	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6211	{
6212	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6213	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6214	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6215	}
6216
6217
6218	/**
6219	* Injects a general-protection (#GP) fault into the VM.
6220	*
6221	* @returns VBox status code (informational status code included).
6222	* @param pVCpu Pointer to the VMCPU.
6223	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6224	* out-of-sync. Make sure to update the required fields
6225	* before using them.
6226	* @param u32ErrorCode The error code associated with the #GP.
6227	*/
6228	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6229	uint32_t *puIntrState)
6230	{
6231	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6232	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6233	if (fErrorCodeValid)
6234	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6235	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6236	puIntrState);
6237	}
6238
6239
6240	/**
6241	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6242	*
6243	* @param pVCpu Pointer to the VMCPU.
6244	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6245	* out-of-sync. Make sure to update the required fields
6246	* before using them.
6247	* @param uVector The software interrupt vector number.
6248	* @param cbInstr The value of RIP that is to be pushed on the guest
6249	* stack.
6250	*/
6251	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6252	{
6253	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6254	if ( uVector == X86_XCPT_BP
6255	\|\| uVector == X86_XCPT_OF)
6256	{
6257	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6258	}
6259	else
6260	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6261	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6262	}
6263
6264
6265	/**
6266	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6267	* stack.
6268	*
6269	* @returns VBox status code (information status code included).
6270	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6271	* @param pVM Pointer to the VM.
6272	* @param pMixedCtx Pointer to the guest-CPU context.
6273	* @param uValue The value to push to the guest stack.
6274	*/
6275	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6276	{
6277	/*
6278	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6279	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6280	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6281	*/
6282	if (pMixedCtx->sp == 1)
6283	return VINF_EM_RESET;
6284	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6285	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6286	AssertRCReturn(rc, rc);
6287	return rc;
6288	}
6289
6290
6291	/**
6292	* Injects an event into the guest upon VM-entry by updating the relevant fields
6293	* in the VM-entry area in the VMCS.
6294	*
6295	* @returns VBox status code (informational error codes included).
6296	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6297	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6298	*
6299	* @param pVCpu Pointer to the VMCPU.
6300	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6301	* be out-of-sync. Make sure to update the required
6302	* fields before using them.
6303	* @param u64IntrInfo The VM-entry interruption-information field.
6304	* @param cbInstr The VM-entry instruction length in bytes (for
6305	* software interrupts, exceptions and privileged
6306	* software exceptions).
6307	* @param u32ErrCode The VM-entry exception error code.
6308	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6309	* @param puIntrState Pointer to the current guest interruptibility-state.
6310	* This interruptibility-state will be updated if
6311	* necessary. This cannot not be NULL.
6312	*
6313	* @remarks No-long-jump zone!!!
6314	* @remarks Requires CR0!
6315	*/
6316	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6317	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6318	{
6319	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6320	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6321	Assert(puIntrState);
6322	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6323
6324	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6325	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6326
6327	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6328	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6329	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6330
6331	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6332
6333	/* We require CR0 to check if the guest is in real-mode. */
6334	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6335	AssertRCReturn(rc, rc);
6336
6337	/*
6338	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6339	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6340	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6341	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6342	*/
6343	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6344	{
6345	PVM pVM = pVCpu->CTX_SUFF(pVM);
6346	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6347	{
6348	Assert(PDMVmmDevHeapIsEnabled(pVM));
6349	Assert(pVM->hm.s.vmx.pRealModeTSS);
6350
6351	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6352	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6353	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6354	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6355	AssertRCReturn(rc, rc);
6356	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6357
6358	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6359	const size_t cbIdtEntry = 4;
6360	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6361	{
6362	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6363	if (uVector == X86_XCPT_DF)
6364	return VINF_EM_RESET;
6365	else if (uVector == X86_XCPT_GP)
6366	{
6367	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6368	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6369	}
6370
6371	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6372	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6373	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6374	}
6375
6376	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6377	uint16_t uGuestIp = pMixedCtx->ip;
6378	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6379	{
6380	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6381	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6382	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6383	}
6384	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6385	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6386
6387	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6388	uint16_t offIdtEntry = 0;
6389	RTSEL selIdtEntry = 0;
6390	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6391	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6392	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6393	AssertRCReturn(rc, rc);
6394
6395	/* Construct the stack frame for the interrupt/exception handler. */
6396	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6397	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6398	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6399	AssertRCReturn(rc, rc);
6400
6401	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6402	if (rc == VINF_SUCCESS)
6403	{
6404	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6405	pMixedCtx->rip = offIdtEntry;
6406	pMixedCtx->cs.Sel = selIdtEntry;
6407	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6408	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6409	&& uVector == X86_XCPT_PF)
6410	{
6411	pMixedCtx->cr2 = GCPtrFaultAddress;
6412	}
6413	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6414	\| HM_CHANGED_GUEST_RIP
6415	\| HM_CHANGED_GUEST_RFLAGS
6416	\| HM_CHANGED_GUEST_RSP;
6417
6418	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6419	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6420	{
6421	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6422	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6423	Log4(("Clearing inhibition due to STI.\n"));
6424	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6425	}
6426	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6427	}
6428	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6429	return rc;
6430	}
6431	else
6432	{
6433	/*
6434	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6435	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6436	*/
6437	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6438	}
6439	}
6440
6441	/* Validate. */
6442	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6443	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6444	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6445
6446	/* Inject. */
6447	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6448	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6449	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6450	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6451
6452	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6453	&& uVector == X86_XCPT_PF)
6454	{
6455	pMixedCtx->cr2 = GCPtrFaultAddress;
6456	}
6457
6458	Log4(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr,
6459	pMixedCtx->cr2));
6460
6461	AssertRCReturn(rc, rc);
6462	return rc;
6463	}
6464
6465
6466	/**
6467	* Enters the VT-x session.
6468	*
6469	* @returns VBox status code.
6470	* @param pVM Pointer to the VM.
6471	* @param pVCpu Pointer to the VMCPU.
6472	* @param pCpu Pointer to the CPU info struct.
6473	*/
6474	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6475	{
6476	AssertPtr(pVM);
6477	AssertPtr(pVCpu);
6478	Assert(pVM->hm.s.vmx.fSupported);
6479	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6480	NOREF(pCpu);
6481
6482	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6483
6484	/* Make sure we're in VMX root mode. */
6485	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6486	if (!(u32HostCR4 & X86_CR4_VMXE))
6487	{
6488	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6489	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6490	}
6491
6492	/* Load the active VMCS as the current one. */
6493	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6494	if (RT_FAILURE(rc))
6495	return rc;
6496
6497	/** @todo this will change with preemption hooks where can can VMRESUME as long
6498	* as we're no preempted. */
6499	pVCpu->hm.s.fResumeVM = false;
6500	return VINF_SUCCESS;
6501	}
6502
6503
6504	/**
6505	* Leaves the VT-x session.
6506	*
6507	* @returns VBox status code.
6508	* @param pVM Pointer to the VM.
6509	* @param pVCpu Pointer to the VMCPU.
6510	* @param pCtx Pointer to the guest-CPU context.
6511	*/
6512	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6513	{
6514	AssertPtr(pVCpu);
6515	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6516	NOREF(pVM);
6517	NOREF(pCtx);
6518
6519	/** @todo this will change with preemption hooks where we only VMCLEAR when
6520	* we are actually going to be preempted, not all the time like we
6521	* currently do. */
6522
6523	/* Restore host-state bits that VT-x only restores partially. */
6524	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6525	{
6526	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6527	/** @todo r=ramshankar: This is broken when
6528	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6529	* VMXRestoreHostState() may unconditionally enables interrupts. */
6530	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6531	#else
6532	Assert(ASMIntAreEnabled());
6533	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6534	#endif
6535	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6536	}
6537
6538	/*
6539	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6540	* and mark the VMCS launch-state as "clear".
6541	*/
6542	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6543	return rc;
6544	}
6545
6546
6547	/**
6548	* Saves the host state in the VMCS host-state.
6549	* Sets up the VM-exit MSR-load area.
6550	*
6551	* The CPU state will be loaded from these fields on every successful VM-exit.
6552	*
6553	* @returns VBox status code.
6554	* @param pVM Pointer to the VM.
6555	* @param pVCpu Pointer to the VMCPU.
6556	*
6557	* @remarks No-long-jump zone!!!
6558	*/
6559	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6560	{
6561	AssertPtr(pVM);
6562	AssertPtr(pVCpu);
6563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6564
6565	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6566
6567	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6568	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6569	return VINF_SUCCESS;
6570
6571	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6572	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6573
6574	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6575	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6576
6577	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6578	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6579
6580	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6581	return rc;
6582	}
6583
6584
6585	/**
6586	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6587	* loaded from these fields on every successful VM-entry.
6588	*
6589	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6590	* Sets up the VM-entry controls.
6591	* Sets up the appropriate VMX non-root function to execute guest code based on
6592	* the guest CPU mode.
6593	*
6594	* @returns VBox status code.
6595	* @param pVM Pointer to the VM.
6596	* @param pVCpu Pointer to the VMCPU.
6597	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6598	* out-of-sync. Make sure to update the required fields
6599	* before using them.
6600	*
6601	* @remarks No-long-jump zone!!!
6602	*/
6603	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6604	{
6605	AssertPtr(pVM);
6606	AssertPtr(pVCpu);
6607	AssertPtr(pMixedCtx);
6608	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6609
6610	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6611
6612	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6613
6614	/* Determine real-on-v86 mode. */
6615	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6616	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6617	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6618	{
6619	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6620	}
6621
6622	/*
6623	* Load the guest-state into the VMCS.
6624	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6625	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6626	*/
6627	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6628	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6629
6630	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6631	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6632
6633	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6634	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6635
6636	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6637	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6638
6639	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6640	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6641	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6642
6643	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6644	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6645
6646	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6647	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6648
6649	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6650	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6651
6652	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6653	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6654	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6655
6656	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6657	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6658
6659	/* Clear the currently unused reserved bits. */
6660	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_VMX_RESERVED1
6661	\| HM_CHANGED_VMX_RESERVED2);
6662
6663	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6664	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6665	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6666
6667	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6668	return rc;
6669	}
6670
6671
6672	/**
6673	* Does the preparations before executing guest code in VT-x.
6674	*
6675	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6676	* recompiler. We must be cautious what we do here regarding committing
6677	* guest-state information into the the VMCS assuming we assuredly execute the
6678	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6679	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6680	* that the recompiler can (and should) use them when it resumes guest
6681	* execution. Otherwise such operations must be done when we can no longer
6682	* exit to ring-3.
6683	*
6684	* @returns VBox status code (informational status codes included).
6685	* @retval VINF_SUCCESS if we can proceed with running the guest.
6686	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6687	* into the guest.
6688	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6689	*
6690	* @param pVM Pointer to the VM.
6691	* @param pVCpu Pointer to the VMCPU.
6692	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6693	* out-of-sync. Make sure to update the required fields
6694	* before using them.
6695	* @param pVmxTransient Pointer to the VMX transient structure.
6696	*
6697	* @remarks Called with preemption disabled.
6698	*/
6699	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6700	{
6701	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6702
6703	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6704	PGMRZDynMapFlushAutoSet(pVCpu);
6705	#endif
6706
6707	/* Check force flag actions that might require us to go back to ring-3. */
6708	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6709	if (rc != VINF_SUCCESS)
6710	return rc;
6711
6712	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6713	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6714	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6715	{
6716	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6717	RTGCPHYS GCPhysApicBase;
6718	GCPhysApicBase = pMixedCtx->msrApicBase;
6719	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6720
6721	/* Unalias any existing mapping. */
6722	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6723	AssertRCReturn(rc, rc);
6724
6725	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6726	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6727	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6728	AssertRCReturn(rc, rc);
6729
6730	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6731	}
6732
6733	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6734	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6735	pVmxTransient->uEFlags = ASMIntDisableFlags();
6736	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6737	{
6738	ASMSetFlags(pVmxTransient->uEFlags);
6739	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6740	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6741	return VINF_EM_RAW_INTERRUPT;
6742	}
6743	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6744	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6745	#endif
6746
6747	/*
6748	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6749	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6750	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6751	*/
6752	/** @todo Rework event evaluation and injection to be completely separate. */
6753	if (TRPMHasTrap(pVCpu))
6754	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6755
6756	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6757	AssertRCReturn(rc, rc);
6758	return rc;
6759	}
6760
6761
6762	/**
6763	* Prepares to run guest code in VT-x and we've committed to doing so. This
6764	* means there is no backing out to ring-3 or anywhere else at this
6765	* point.
6766	*
6767	* @param pVM Pointer to the VM.
6768	* @param pVCpu Pointer to the VMCPU.
6769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6770	* out-of-sync. Make sure to update the required fields
6771	* before using them.
6772	* @param pVmxTransient Pointer to the VMX transient structure.
6773	*
6774	* @remarks Called with preemption disabled.
6775	* @remarks No-long-jump zone!!!
6776	*/
6777	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6778	{
6779	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6780	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6781
6782	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6783	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6784	pVmxTransient->uEFlags = ASMIntDisableFlags();
6785	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6786	#endif
6787
6788	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6789	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6790	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6791	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6792	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6793	#endif
6794	int rc = VINF_SUCCESS;
6795	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6796	{
6797	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6798	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6799	}
6800	else if (pVCpu->hm.s.fContextUseFlags)
6801	{
6802	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6803	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6804	}
6805	AssertRC(rc);
6806	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6807
6808	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6809	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6810	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6811
6812	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6813	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6814	{
6815	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6816	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6817	}
6818
6819	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6820	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6821	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6822
6823	/*
6824	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6825	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6826	*/
6827	if (pVM->hm.s.fTPRPatchingActive)
6828	{
6829	Assert(!CPUMIsGuestInLongMode(pVCpu));
6830
6831	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6832	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6833	AssertRC(rc);
6834
6835	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6836	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6837	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6838	see hmR0VmxLoadGuestApicState(). */
6839	}
6840
6841	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6842	/*
6843	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6844	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6845	*/
6846	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6847	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6848	{
6849	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6850	uint64_t u64HostTscAux = 0;
6851	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6852	AssertRC(rc2);
6853	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6854	}
6855	#endif
6856
6857	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6858	to start executing. */
6859	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6860	}
6861
6862
6863	/**
6864	* Performs some essential restoration of state after running guest code in
6865	* VT-x.
6866	*
6867	* @param pVM Pointer to the VM.
6868	* @param pVCpu Pointer to the VMCPU.
6869	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6870	* out-of-sync. Make sure to update the required fields
6871	* before using them.
6872	* @param pVmxTransient Pointer to the VMX transient structure.
6873	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6874	*
6875	* @remarks Called with interrupts disabled.
6876	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6877	* unconditionally when it is safe to do so.
6878	*/
6879	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6880	{
6881	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6882	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6883
6884	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6885	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6886	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6887	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6888	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6889
6890	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6891	{
6892	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6893	/* Restore host's TSC_AUX. */
6894	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6895	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6896	#endif
6897	/** @todo Find a way to fix hardcoding a guestimate. */
6898	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6899	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6900	}
6901
6902	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6903	Assert(!(ASMGetFlags() & X86_EFL_IF));
6904	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6905
6906	/* Restore the effects of TPR patching if any. */
6907	if (pVM->hm.s.fTPRPatchingActive)
6908	{
6909	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6910	AssertRC(rc);
6911	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6912	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6913	}
6914
6915	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6916	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6917
6918	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6919	uint32_t uExitReason;
6920	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6921	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6922	AssertRC(rc);
6923	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6924	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6925
6926	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6927	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6928
6929	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6930	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6931	{
6932	Log4(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6933	return;
6934	}
6935
6936	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6937	{
6938	/* Update the guest interruptibility-state from the VMCS. */
6939	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6940	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6941	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6942	AssertRC(rc);
6943	#endif
6944	/*
6945	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6946	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6947	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6948	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6949	*/
6950	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6951	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6952	{
6953	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6954	AssertRC(rc);
6955	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6956	}
6957	}
6958	}
6959
6960
6961	/**
6962	* Runs the guest code using VT-x.
6963	*
6964	* @returns VBox status code.
6965	* @param pVM Pointer to the VM.
6966	* @param pVCpu Pointer to the VMCPU.
6967	* @param pCtx Pointer to the guest-CPU context.
6968	*
6969	* @remarks Called with preemption disabled.
6970	*/
6971	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6972	{
6973	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6974	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6975
6976	VMXTRANSIENT VmxTransient;
6977	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6978	int rc = VERR_INTERNAL_ERROR_5;
6979	uint32_t cLoops = 0;
6980
6981	for (;; cLoops++)
6982	{
6983	Assert(!HMR0SuspendPending());
6984	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6985	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6986	(unsigned)RTMpCpuId(), cLoops));
6987
6988	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6989	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6990	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6991	if (rc != VINF_SUCCESS)
6992	break;
6993
6994	/*
6995	* No longjmps to ring-3 from this point on!!!
6996	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6997	* This also disables flushing of the R0-logger instance (if any).
6998	*/
6999	VMMRZCallRing3Disable(pVCpu);
7000	VMMRZCallRing3RemoveNotification(pVCpu);
7001	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7002
7003	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7004	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7005
7006	/*
7007	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7008	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7009	*/
7010	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7011	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7012	{
7013	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7014	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7015	return rc;
7016	}
7017
7018	/* Handle the VM-exit. */
7019	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7020	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7021	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7022	HMVMX_START_EXIT_DISPATCH_PROF();
7023	#ifdef HMVMX_USE_FUNCTION_TABLE
7024	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7025	#else
7026	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7027	#endif
7028	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7029	if (rc != VINF_SUCCESS)
7030	break;
7031	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7032	{
7033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7034	rc = VINF_EM_RAW_INTERRUPT;
7035	break;
7036	}
7037	}
7038
7039	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7040	if (rc == VERR_EM_INTERPRETER)
7041	rc = VINF_EM_RAW_EMULATE_INSTR;
7042	else if (rc == VINF_EM_RESET)
7043	rc = VINF_EM_TRIPLE_FAULT;
7044	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7045	return rc;
7046	}
7047
7048
7049	#ifndef HMVMX_USE_FUNCTION_TABLE
7050	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7051	{
7052	int rc;
7053	switch (rcReason)
7054	{
7055	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7056	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7057	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7058	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7059	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7060	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7061	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7062	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7063	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7064	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7065	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7066	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7067	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7068	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7069	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7070	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7071	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7072	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7073	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7074	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7075	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7076	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7077	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7078	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7079	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7080	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7081	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7082	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7083	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7084	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7085	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7086	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7087	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7088
7089	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7090	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7091	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7092	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7093	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7094	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7095	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7096	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7097	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7098
7099	case VMX_EXIT_VMCALL:
7100	case VMX_EXIT_VMCLEAR:
7101	case VMX_EXIT_VMLAUNCH:
7102	case VMX_EXIT_VMPTRLD:
7103	case VMX_EXIT_VMPTRST:
7104	case VMX_EXIT_VMREAD:
7105	case VMX_EXIT_VMRESUME:
7106	case VMX_EXIT_VMWRITE:
7107	case VMX_EXIT_VMXOFF:
7108	case VMX_EXIT_VMXON:
7109	case VMX_EXIT_INVEPT:
7110	case VMX_EXIT_INVVPID:
7111	case VMX_EXIT_VMFUNC:
7112	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7113	break;
7114	default:
7115	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7116	break;
7117	}
7118	return rc;
7119	}
7120	#endif
7121
7122	#ifdef DEBUG
7123	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7124	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7125	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7126	# define VMX_ASSERT_PREEMPT_CPUID() \
7127	do \
7128	{ \
7129	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7130	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7131	} while (0)
7132
7133	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7134	do { \
7135	AssertPtr(pVCpu); \
7136	AssertPtr(pMixedCtx); \
7137	AssertPtr(pVmxTransient); \
7138	Assert(pVmxTransient->fVMEntryFailed == false); \
7139	Assert(ASMIntAreEnabled()); \
7140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7141	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7142	Log4Func(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7143	(unsigned)pVCpu->idCpu)); \
7144	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7145	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7146	VMX_ASSERT_PREEMPT_CPUID(); \
7147	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7148	} while (0)
7149	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7150	do { \
7151	Log4Func(("\n")); \
7152	} while(0)
7153	#else /* Release builds */
7154	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7155	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7156	#endif
7157
7158
7159	/**
7160	* Advances the guest RIP after reading it from the VMCS.
7161	*
7162	* @returns VBox status code.
7163	* @param pVCpu Pointer to the VMCPU.
7164	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7165	* out-of-sync. Make sure to update the required fields
7166	* before using them.
7167	* @param pVmxTransient Pointer to the VMX transient structure.
7168	*
7169	* @remarks No-long-jump zone!!!
7170	*/
7171	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7172	{
7173	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7174	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7175	AssertRCReturn(rc, rc);
7176
7177	pMixedCtx->rip += pVmxTransient->cbInstr;
7178	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7179	return rc;
7180	}
7181
7182
7183	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7184	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7185	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7186	/**
7187	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7188	*/
7189	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7190	{
7191	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7193	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7194	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7195	Assert(ASMIntAreEnabled());
7196	return VINF_SUCCESS;
7197	#else
7198	return VINF_EM_RAW_INTERRUPT;
7199	#endif
7200	}
7201
7202
7203	/**
7204	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7205	*/
7206	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7207	{
7208	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7209	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7210
7211	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7212	AssertRCReturn(rc, rc);
7213
7214	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7215	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7216	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7217
7218	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7219	{
7220	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7221	return VINF_EM_RAW_INTERRUPT;
7222	}
7223
7224	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7225	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7226	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7227	{
7228	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7229	return VINF_SUCCESS;
7230	}
7231	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7232	{
7233	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7234	return rc;
7235	}
7236
7237	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7238	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7239	switch (uIntrType)
7240	{
7241	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7242	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7243	/* no break */
7244	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7245	{
7246	switch (uVector)
7247	{
7248	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7249	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7250	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7251	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7252	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7253	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7254	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7255	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7256	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7257	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7258	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7259	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7260	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7261	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7262	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7263	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7264	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7265	#endif
7266	default:
7267	{
7268	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7269	AssertRCReturn(rc, rc);
7270
7271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7272	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7273	{
7274	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7275	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7276	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7277	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7278	AssertRCReturn(rc, rc);
7279	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7280	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7281	0 /* GCPtrFaultAddress */);
7282	AssertRCReturn(rc, rc);
7283	}
7284	else
7285	{
7286	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7287	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7288	}
7289	break;
7290	}
7291	}
7292	break;
7293	}
7294
7295	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7296	default:
7297	{
7298	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7299	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7300	break;
7301	}
7302	}
7303	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7304	return rc;
7305	}
7306
7307
7308	/**
7309	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7310	*/
7311	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7312	{
7313	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7314
7315	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7316	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7317	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7318	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7319	AssertRCReturn(rc, rc);
7320
7321	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7323	return VINF_SUCCESS;
7324	}
7325
7326
7327	/**
7328	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7329	*/
7330	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7331	{
7332	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7333	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7334	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7335	}
7336
7337
7338	/**
7339	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7340	*/
7341	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7342	{
7343	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7344	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7345	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7346	}
7347
7348
7349	/**
7350	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7351	*/
7352	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7353	{
7354	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7356	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7357	}
7358
7359
7360	/**
7361	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7362	*/
7363	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7364	{
7365	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7366	PVM pVM = pVCpu->CTX_SUFF(pVM);
7367	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7368	if (RT_LIKELY(rc == VINF_SUCCESS))
7369	{
7370	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7371	Assert(pVmxTransient->cbInstr == 2);
7372	}
7373	else
7374	{
7375	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7376	rc = VERR_EM_INTERPRETER;
7377	}
7378	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7379	return rc;
7380	}
7381
7382
7383	/**
7384	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7385	*/
7386	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7387	{
7388	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7389	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7390	AssertRCReturn(rc, rc);
7391
7392	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7393	return VINF_EM_RAW_EMULATE_INSTR;
7394
7395	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7396	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7397	}
7398
7399
7400	/**
7401	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7402	*/
7403	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7404	{
7405	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7406	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7407	AssertRCReturn(rc, rc);
7408
7409	PVM pVM = pVCpu->CTX_SUFF(pVM);
7410	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7411	if (RT_LIKELY(rc == VINF_SUCCESS))
7412	{
7413	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7414	Assert(pVmxTransient->cbInstr == 2);
7415	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7416	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7417	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7418	}
7419	else
7420	{
7421	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7422	rc = VERR_EM_INTERPRETER;
7423	}
7424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7425	return rc;
7426	}
7427
7428
7429	/**
7430	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7431	*/
7432	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7433	{
7434	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7435	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7436	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7437	AssertRCReturn(rc, rc);
7438
7439	PVM pVM = pVCpu->CTX_SUFF(pVM);
7440	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7441	if (RT_LIKELY(rc == VINF_SUCCESS))
7442	{
7443	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7444	Assert(pVmxTransient->cbInstr == 3);
7445	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7446	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7447	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7448	}
7449	else
7450	{
7451	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7452	rc = VERR_EM_INTERPRETER;
7453	}
7454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7455	return rc;
7456	}
7457
7458
7459	/**
7460	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7461	*/
7462	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7465	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7466	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7467	AssertRCReturn(rc, rc);
7468
7469	PVM pVM = pVCpu->CTX_SUFF(pVM);
7470	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7471	if (RT_LIKELY(rc == VINF_SUCCESS))
7472	{
7473	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7474	Assert(pVmxTransient->cbInstr == 2);
7475	}
7476	else
7477	{
7478	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7479	rc = VERR_EM_INTERPRETER;
7480	}
7481	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7482	return rc;
7483	}
7484
7485
7486	/**
7487	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7488	*/
7489	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7490	{
7491	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7492	PVM pVM = pVCpu->CTX_SUFF(pVM);
7493	Assert(!pVM->hm.s.fNestedPaging);
7494
7495	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7496	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7497	AssertRCReturn(rc, rc);
7498
7499	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7500	rc = VBOXSTRICTRC_VAL(rc2);
7501	if (RT_LIKELY(rc == VINF_SUCCESS))
7502	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7503	else
7504	{
7505	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7506	pVmxTransient->uExitQualification, rc));
7507	}
7508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7509	return rc;
7510	}
7511
7512
7513	/**
7514	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7515	*/
7516	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7517	{
7518	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7519	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7520	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7521	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7522	AssertRCReturn(rc, rc);
7523
7524	PVM pVM = pVCpu->CTX_SUFF(pVM);
7525	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7526	if (RT_LIKELY(rc == VINF_SUCCESS))
7527	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7528	else
7529	{
7530	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7531	rc = VERR_EM_INTERPRETER;
7532	}
7533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7534	return rc;
7535	}
7536
7537
7538	/**
7539	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7540	*/
7541	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7542	{
7543	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7544	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7545	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7546	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7547	AssertRCReturn(rc, rc);
7548
7549	PVM pVM = pVCpu->CTX_SUFF(pVM);
7550	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7551	rc = VBOXSTRICTRC_VAL(rc2);
7552	if (RT_LIKELY( rc == VINF_SUCCESS
7553	\|\| rc == VINF_EM_HALT))
7554	{
7555	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7556	AssertRCReturn(rc3, rc3);
7557
7558	if ( rc == VINF_EM_HALT
7559	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7560	{
7561	rc = VINF_SUCCESS;
7562	}
7563	}
7564	else
7565	{
7566	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7567	rc = VERR_EM_INTERPRETER;
7568	}
7569	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7570	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7571	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7572	return rc;
7573	}
7574
7575
7576	/**
7577	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7578	*/
7579	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7580	{
7581	/*
7582	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7583	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7584	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7585	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7586	*/
7587	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7588	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7589	}
7590
7591
7592	/**
7593	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7594	*/
7595	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7596	{
7597	/*
7598	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7599	* root operation. If we get there there is something funny going on.
7600	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7601	*/
7602	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7603	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7604	}
7605
7606
7607	/**
7608	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7609	*/
7610	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7611	{
7612	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7613	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7614	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7615	}
7616
7617
7618	/**
7619	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7620	*/
7621	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7622	{
7623	/*
7624	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7625	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7626	* See Intel spec. 25.3 "Other Causes of VM-exits".
7627	*/
7628	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7629	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7630	}
7631
7632
7633	/**
7634	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7635	* VM-exit.
7636	*/
7637	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7638	{
7639	/*
7640	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7641	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7642	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7643	*/
7644	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7645	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7646	}
7647
7648
7649	/**
7650	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7651	* VM-exit.
7652	*/
7653	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7654	{
7655	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7656	return VINF_EM_RESET;
7657	}
7658
7659
7660	/**
7661	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7662	*/
7663	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7664	{
7665	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7666	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7667	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7668	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7669	AssertRCReturn(rc, rc);
7670
7671	pMixedCtx->rip++;
7672	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7673	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7674	rc = VINF_SUCCESS;
7675	else
7676	rc = VINF_EM_HALT;
7677
7678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7679	return rc;
7680	}
7681
7682
7683	/**
7684	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7685	*/
7686	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7687	{
7688	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7689	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7690	return VINF_SUCCESS;
7691	}
7692
7693
7694	/**
7695	* VM-exit handler for expiry of the VMX preemption timer.
7696	*/
7697	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7698	{
7699	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7700
7701	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7702	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7703
7704	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7705	PVM pVM = pVCpu->CTX_SUFF(pVM);
7706	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7708	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7709	}
7710
7711
7712	/**
7713	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7714	*/
7715	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7716	{
7717	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7718
7719	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7720	/** @todo check if XSETBV is supported by the recompiler. */
7721	return VERR_EM_INTERPRETER;
7722	}
7723
7724
7725	/**
7726	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7727	*/
7728	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7729	{
7730	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7731
7732	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7733	/** @todo implement EMInterpretInvpcid() */
7734	return VERR_EM_INTERPRETER;
7735	}
7736
7737
7738	/**
7739	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7740	* Error VM-exit.
7741	*/
7742	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7743	{
7744	uint32_t uIntrState;
7745	HMVMXHCUINTREG uHCReg;
7746	uint64_t u64Val;
7747	uint32_t u32Val;
7748
7749	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7750	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7751	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7752	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7753	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7754	AssertRCReturn(rc, rc);
7755
7756	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7757	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7758	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7759	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7760
7761	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7762	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7763	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7764	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7765	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7766	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7767	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7768	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7769	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7770	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7771	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7772	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7773
7774	PVM pVM = pVCpu->CTX_SUFF(pVM);
7775	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7776
7777	return VERR_VMX_INVALID_GUEST_STATE;
7778	}
7779
7780
7781	/**
7782	* VM-exit handler for VM-entry failure due to an MSR-load
7783	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7784	*/
7785	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7786	{
7787	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7788	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7789	}
7790
7791
7792	/**
7793	* VM-exit handler for VM-entry failure due to a machine-check event
7794	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7795	*/
7796	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7797	{
7798	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7799	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7800	}
7801
7802
7803	/**
7804	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7805	* theory.
7806	*/
7807	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7808	{
7809	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7810	return VERR_VMX_UNDEFINED_EXIT_CODE;
7811	}
7812
7813
7814	/**
7815	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7816	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7817	* Conditional VM-exit.
7818	*/
7819	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7820	{
7821	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7822
7823	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7824	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7825	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7826	return VERR_EM_INTERPRETER;
7827	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7828	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7829	}
7830
7831
7832	/**
7833	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7834	*/
7835	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7836	{
7837	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7838
7839	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7840	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7841	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7842	return VERR_EM_INTERPRETER;
7843	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7844	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7845	}
7846
7847
7848	/**
7849	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7850	*/
7851	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7852	{
7853	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7854
7855	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7856	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7857	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7858	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7859	AssertRCReturn(rc, rc);
7860
7861	PVM pVM = pVCpu->CTX_SUFF(pVM);
7862	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7863	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7864	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7866
7867	if (RT_LIKELY(rc == VINF_SUCCESS))
7868	{
7869	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7870	Assert(pVmxTransient->cbInstr == 2);
7871	}
7872	return rc;
7873	}
7874
7875
7876	/**
7877	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7878	*/
7879	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7880	{
7881	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7882	PVM pVM = pVCpu->CTX_SUFF(pVM);
7883	int rc = VINF_SUCCESS;
7884
7885	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7886	if ( pVM->hm.s.fTPRPatchingActive
7887	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7888	{
7889	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7890	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7891	{
7892	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7893	AssertRC(rc);
7894	}
7895
7896	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7897	Assert(pVmxTransient->cbInstr == 2);
7898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7899	return VINF_SUCCESS;
7900	}
7901
7902	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7903	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7904	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7905	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7906	AssertRCReturn(rc, rc);
7907	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
7908
7909	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7910	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7912
7913	if (RT_LIKELY(rc == VINF_SUCCESS))
7914	{
7915	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7916
7917	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7918	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7919	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7920	{
7921	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7922	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7923	}
7924	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7925	{
7926	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7927	AssertRCReturn(rc, rc);
7928	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7929	}
7930	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7931	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7932
7933	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7934	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7935	{
7936	switch (pMixedCtx->ecx)
7937	{
7938	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7939	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7940	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7941	case MSR_K8_FS_BASE: /* no break */
7942	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7943	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7944	}
7945	}
7946	#ifdef VBOX_STRICT
7947	else
7948	{
7949	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7950	switch (pMixedCtx->ecx)
7951	{
7952	case MSR_IA32_SYSENTER_CS:
7953	case MSR_IA32_SYSENTER_EIP:
7954	case MSR_IA32_SYSENTER_ESP:
7955	case MSR_K8_FS_BASE:
7956	case MSR_K8_GS_BASE:
7957	{
7958	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7959	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7960	}
7961
7962	case MSR_K8_LSTAR:
7963	case MSR_K6_STAR:
7964	case MSR_K8_SF_MASK:
7965	case MSR_K8_TSC_AUX:
7966	case MSR_K8_KERNEL_GS_BASE:
7967	{
7968	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7969	pMixedCtx->ecx));
7970	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7971	}
7972	}
7973	}
7974	#endif /* VBOX_STRICT */
7975	}
7976	return rc;
7977	}
7978
7979
7980	/**
7981	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7982	*/
7983	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7984	{
7985	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7986
7987	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7988	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7989	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7990	return VERR_EM_INTERPRETER;
7991	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7992	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7993	}
7994
7995
7996	/**
7997	* VM-exit handler for when the TPR value is lowered below the specified
7998	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7999	*/
8000	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8001	{
8002	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8003	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8004
8005	/*
8006	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8007	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8008	* resume guest execution.
8009	*/
8010	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8012	return VINF_SUCCESS;
8013	}
8014
8015
8016	/**
8017	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8018	* VM-exit.
8019	*
8020	* @retval VINF_SUCCESS when guest execution can continue.
8021	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8022	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8023	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8024	* recompiler.
8025	*/
8026	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8027	{
8028	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8029	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8030	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8031	AssertRCReturn(rc, rc);
8032
8033	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8034	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8035	PVM pVM = pVCpu->CTX_SUFF(pVM);
8036	switch (uAccessType)
8037	{
8038	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8039	{
8040	#if 0
8041	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8042	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8043	#else
8044	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8045	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8046	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8047	#endif
8048	AssertRCReturn(rc, rc);
8049
8050	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8051	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8052	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8053	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8054
8055	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8056	{
8057	case 0: /* CR0 */
8058	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8059	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8060	break;
8061	case 2: /* C2 **/
8062	/* Nothing to do here, CR2 it's not part of the VMCS. */
8063	break;
8064	case 3: /* CR3 */
8065	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8066	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8067	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8068	break;
8069	case 4: /* CR4 */
8070	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8071	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8072	break;
8073	case 8: /* CR8 */
8074	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8075	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8076	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8077	break;
8078	default:
8079	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8080	break;
8081	}
8082
8083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8084	break;
8085	}
8086
8087	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8088	{
8089	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8090	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8091	AssertRCReturn(rc, rc);
8092	Assert( !pVM->hm.s.fNestedPaging
8093	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8094	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8095
8096	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8097	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8098	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8099
8100	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8101	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8102	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8103	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8104	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8105	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8106	break;
8107	}
8108
8109	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8110	{
8111	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8112	AssertRCReturn(rc, rc);
8113	rc = EMInterpretCLTS(pVM, pVCpu);
8114	AssertRCReturn(rc, rc);
8115	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8117	Log4(("CRX CLTS write rc=%d\n", rc));
8118	break;
8119	}
8120
8121	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8122	{
8123	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8124	AssertRCReturn(rc, rc);
8125	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8126	if (RT_LIKELY(rc == VINF_SUCCESS))
8127	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8128	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8129	Log4(("CRX LMSW write rc=%d\n", rc));
8130	break;
8131	}
8132
8133	default:
8134	{
8135	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8136	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8137	}
8138	}
8139
8140	/* Validate possible error codes. */
8141	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8142	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8143	if (RT_SUCCESS(rc))
8144	{
8145	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8146	AssertRCReturn(rc2, rc2);
8147	}
8148
8149	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8150	return rc;
8151	}
8152
8153
8154	/**
8155	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8156	* VM-exit.
8157	*/
8158	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8159	{
8160	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8161	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8162
8163	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8164	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8165	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8166	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8167	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8168	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8169	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8170	AssertRCReturn(rc, rc);
8171	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8172
8173	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8174	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8175	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8176	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8177	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8178	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8179	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8180
8181	/* I/O operation lookup arrays. */
8182	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8183	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8184
8185	const uint32_t cbSize = s_aIOSize[uIOWidth];
8186	const uint32_t cbInstr = pVmxTransient->cbInstr;
8187	PVM pVM = pVCpu->CTX_SUFF(pVM);
8188	if (fIOString)
8189	{
8190	/* INS/OUTS - I/O String instruction. */
8191	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8192	/** @todo for now manually disassemble later optimize by getting the fields from
8193	* the VMCS. */
8194	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8195	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8196	* segment prefix info. */
8197	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8198	if (RT_SUCCESS(rc))
8199	{
8200	if (fIOWrite)
8201	{
8202	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8203	(DISCPUMODE)pDis->uAddrMode, cbSize);
8204	rc = VBOXSTRICTRC_VAL(rc2);
8205	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8206	}
8207	else
8208	{
8209	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8210	(DISCPUMODE)pDis->uAddrMode, cbSize);
8211	rc = VBOXSTRICTRC_VAL(rc2);
8212	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8213	}
8214	}
8215	else
8216	{
8217	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8218	rc = VINF_EM_RAW_EMULATE_INSTR;
8219	}
8220	}
8221	else
8222	{
8223	/* IN/OUT - I/O instruction. */
8224	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8225	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8226	if (fIOWrite)
8227	{
8228	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8229	rc = VBOXSTRICTRC_VAL(rc2);
8230	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8231	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8232	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8233	}
8234	else
8235	{
8236	uint32_t u32Result = 0;
8237	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8238	rc = VBOXSTRICTRC_VAL(rc2);
8239	if (IOM_SUCCESS(rc))
8240	{
8241	/* Save result of I/O IN instr. in AL/AX/EAX. */
8242	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8243	}
8244	else if (rc == VINF_IOM_R3_IOPORT_READ)
8245	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8247	}
8248	}
8249
8250	if (IOM_SUCCESS(rc))
8251	{
8252	pMixedCtx->rip += cbInstr;
8253	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8254	if (RT_LIKELY(rc == VINF_SUCCESS))
8255	{
8256	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8257	AssertRCReturn(rc, rc);
8258
8259	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8260	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8261	{
8262	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8263	for (unsigned i = 0; i < 4; i++)
8264	{
8265	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8266	if ( ( uIOPort >= pMixedCtx->dr[i]
8267	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8268	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8269	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8270	{
8271	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8272	uint64_t uDR6 = ASMGetDR6();
8273
8274	/* Clear all breakpoint status flags and set the one we just hit. */
8275	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8276	uDR6 \|= (uint64_t)RT_BIT(i);
8277
8278	/*
8279	* Note: AMD64 Architecture Programmer's Manual 13.1:
8280	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8281	* be cleared by software after the contents have been read.
8282	*/
8283	ASMSetDR6(uDR6);
8284
8285	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8286	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8287
8288	/* Paranoia. */
8289	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* Upper 32 bits MBZ. */
8290	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8291	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8292
8293	/* Resync DR7 */
8294	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8295	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8296
8297	/* Set #DB to be injected into the VM and continue guest execution. */
8298	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8299	break;
8300	}
8301	}
8302	}
8303	}
8304	}
8305
8306	#ifdef DEBUG
8307	if (rc == VINF_IOM_R3_IOPORT_READ)
8308	Assert(!fIOWrite);
8309	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8310	Assert(fIOWrite);
8311	else
8312	{
8313	AssertMsg( RT_FAILURE(rc)
8314	\|\| rc == VINF_SUCCESS
8315	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8316	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8317	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8318	}
8319	#endif
8320
8321	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8322	return rc;
8323	}
8324
8325
8326	/**
8327	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8328	* VM-exit.
8329	*/
8330	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8331	{
8332	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8333
8334	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8335	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8336	AssertRCReturn(rc, rc);
8337	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8338	{
8339	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8340	AssertRCReturn(rc, rc);
8341	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8342	{
8343	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8344	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8345	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8346	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8347	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8348	{
8349	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8350	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8351
8352	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8353	Assert(!pVCpu->hm.s.Event.fPending);
8354	pVCpu->hm.s.Event.fPending = true;
8355	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8356	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8357	AssertRCReturn(rc, rc);
8358	if (fErrorCodeValid)
8359	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8360	else
8361	pVCpu->hm.s.Event.u32ErrCode = 0;
8362	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8363	&& uVector == X86_XCPT_PF)
8364	{
8365	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8366	}
8367	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8368	}
8369	}
8370	}
8371	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8372	* emulation. */
8373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8374	return VERR_EM_INTERPRETER;
8375	}
8376
8377
8378	/**
8379	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8380	*/
8381	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8382	{
8383	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8384	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8385	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8386	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8387	AssertRCReturn(rc, rc);
8388	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8389	return VINF_EM_DBG_STOP;
8390	}
8391
8392
8393	/**
8394	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8395	*/
8396	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8397	{
8398	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8399
8400	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8401	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8402	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8403	return VINF_SUCCESS;
8404	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8405	return rc;
8406
8407	#if 0
8408	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8409	* just sync the whole thing. */
8410	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8411	#else
8412	/* Aggressive state sync. for now. */
8413	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8414	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8415	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8416	#endif
8417	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8418	AssertRCReturn(rc, rc);
8419
8420	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8421	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8422	switch (uAccessType)
8423	{
8424	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8425	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8426	{
8427	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8428	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8429	{
8430	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8431	}
8432
8433	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8434	GCPhys &= PAGE_BASE_GC_MASK;
8435	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8436	PVM pVM = pVCpu->CTX_SUFF(pVM);
8437	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8438	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8439
8440	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8441	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8442	CPUMCTX2CORE(pMixedCtx), GCPhys);
8443	rc = VBOXSTRICTRC_VAL(rc2);
8444	Log4(("ApicAccess rc=%d\n", rc));
8445	if ( rc == VINF_SUCCESS
8446	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8447	\|\| rc == VERR_PAGE_NOT_PRESENT)
8448	{
8449	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8450	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8451	rc = VINF_SUCCESS;
8452	}
8453	break;
8454	}
8455
8456	default:
8457	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8458	rc = VINF_EM_RAW_EMULATE_INSTR;
8459	break;
8460	}
8461
8462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8463	return rc;
8464	}
8465
8466
8467	/**
8468	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8469	* VM-exit.
8470	*/
8471	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8472	{
8473	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8474
8475	/* We should -not- get this VM-exit if the guest is debugging. */
8476	if (CPUMIsGuestDebugStateActive(pVCpu))
8477	{
8478	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8479	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8480	}
8481
8482	int rc = VERR_INTERNAL_ERROR_5;
8483	if ( !DBGFIsStepping(pVCpu)
8484	&& !CPUMIsHyperDebugStateActive(pVCpu))
8485	{
8486	/* Don't intercept MOV DRx. */
8487	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8488	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8489	AssertRCReturn(rc, rc);
8490
8491	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8492	PVM pVM = pVCpu->CTX_SUFF(pVM);
8493	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8494	AssertRC(rc);
8495	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8496
8497	#ifdef VBOX_WITH_STATISTICS
8498	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8499	AssertRCReturn(rc, rc);
8500	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8501	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8502	else
8503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8504	#endif
8505	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8506	return VINF_SUCCESS;
8507	}
8508
8509	/*
8510	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8511	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8512	*/
8513	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8514	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8515	AssertRCReturn(rc, rc);
8516
8517	PVM pVM = pVCpu->CTX_SUFF(pVM);
8518	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8519	{
8520	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8521	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8522	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8523	if (RT_SUCCESS(rc))
8524	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8526	}
8527	else
8528	{
8529	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8530	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8531	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8532	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8533	}
8534
8535	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8536	if (RT_SUCCESS(rc))
8537	{
8538	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8539	AssertRCReturn(rc2, rc2);
8540	}
8541	return rc;
8542	}
8543
8544
8545	/**
8546	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8547	* Conditional VM-exit.
8548	*/
8549	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8550	{
8551	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8552	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8553
8554	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8555	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8556	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8557	return VINF_SUCCESS;
8558	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8559	return rc;
8560
8561	RTGCPHYS GCPhys = 0;
8562	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8563
8564	#if 0
8565	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8566	#else
8567	/* Aggressive state sync. for now. */
8568	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8569	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8570	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8571	#endif
8572	AssertRCReturn(rc, rc);
8573
8574	/*
8575	* If we succeed, resume guest execution.
8576	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8577	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8578	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8579	* weird case. See @bugref{6043}.
8580	*/
8581	PVM pVM = pVCpu->CTX_SUFF(pVM);
8582	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8583	rc = VBOXSTRICTRC_VAL(rc2);
8584	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8585	if ( rc == VINF_SUCCESS
8586	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8587	\|\| rc == VERR_PAGE_NOT_PRESENT)
8588	{
8589	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8590	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8591	return VINF_SUCCESS;
8592	}
8593	return rc;
8594	}
8595
8596
8597	/**
8598	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8599	* VM-exit.
8600	*/
8601	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8602	{
8603	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8604	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8605
8606	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8607	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8608	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8609	return VINF_SUCCESS;
8610	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8611	return rc;
8612
8613	RTGCPHYS GCPhys = 0;
8614	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8615	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8616	#if 0
8617	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8618	#else
8619	/* Aggressive state sync. for now. */
8620	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8621	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8622	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8623	#endif
8624	AssertRCReturn(rc, rc);
8625
8626	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8627	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8628
8629	RTGCUINT uErrorCode = 0;
8630	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8631	uErrorCode \|= X86_TRAP_PF_ID;
8632	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8633	uErrorCode \|= X86_TRAP_PF_RW;
8634	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8635	uErrorCode \|= X86_TRAP_PF_P;
8636
8637	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8638
8639	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8640	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8641
8642	/* Handle the pagefault trap for the nested shadow table. */
8643	PVM pVM = pVCpu->CTX_SUFF(pVM);
8644	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8645	TRPMResetTrap(pVCpu);
8646
8647	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8648	if ( rc == VINF_SUCCESS
8649	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8650	\|\| rc == VERR_PAGE_NOT_PRESENT)
8651	{
8652	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8654	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8655	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8656	return VINF_SUCCESS;
8657	}
8658
8659	Log4(("EPT return to ring-3 rc=%d\n"));
8660	return rc;
8661	}
8662
8663
8664	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8665	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8666	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8667	/**
8668	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8669	*/
8670	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8671	{
8672	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8674
8675	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8676	AssertRCReturn(rc, rc);
8677
8678	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8679	{
8680	/* Old-style FPU error reporting needs some extra work. */
8681	/** @todo don't fall back to the recompiler, but do it manually. */
8682	return VERR_EM_INTERPRETER;
8683	}
8684	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8685	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8686	return rc;
8687	}
8688
8689
8690	/**
8691	* VM-exit exception handler for #BP (Breakpoint exception).
8692	*/
8693	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8694	{
8695	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8696	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8697
8698	/** @todo Try optimize this by not saving the entire guest state unless
8699	* really needed. */
8700	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8701	AssertRCReturn(rc, rc);
8702
8703	PVM pVM = pVCpu->CTX_SUFF(pVM);
8704	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8705	if (rc == VINF_EM_RAW_GUEST_TRAP)
8706	{
8707	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8708	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8709	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8710	AssertRCReturn(rc, rc);
8711
8712	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8713	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8714	}
8715
8716	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8717	return rc;
8718	}
8719
8720
8721	/**
8722	* VM-exit exception handler for #DB (Debug exception).
8723	*/
8724	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8725	{
8726	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8727	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8728
8729	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8730	AssertRCReturn(rc, rc);
8731
8732	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8733	uint64_t uDR6 = X86_DR6_INIT_VAL;
8734	uDR6 \|= (pVmxTransient->uExitQualification
8735	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8736	PVM pVM = pVCpu->CTX_SUFF(pVM);
8737	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8738	if (rc == VINF_EM_RAW_GUEST_TRAP)
8739	{
8740	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8741	pMixedCtx->dr[6] = uDR6;
8742
8743	if (CPUMIsGuestDebugStateActive(pVCpu))
8744	ASMSetDR6(pMixedCtx->dr[6]);
8745
8746	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8747
8748	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8749	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8750
8751	/* Paranoia. */
8752	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* upper 32 bits MBZ. */
8753	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8754	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8755
8756	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8757	AssertRCReturn(rc,rc);
8758
8759	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8760	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8761	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8762	AssertRCReturn(rc2, rc2);
8763	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8764	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8765	rc = VINF_SUCCESS;
8766	}
8767
8768	return rc;
8769	}
8770
8771
8772	/**
8773	* VM-exit exception handler for #NM (Device-not-available exception: floating
8774	* point exception).
8775	*/
8776	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8777	{
8778	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8779
8780	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8781	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8782	#endif
8783
8784	/* We require CR0 and EFER. EFER is always up-to-date. */
8785	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8786	AssertRCReturn(rc, rc);
8787
8788	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8789	PVM pVM = pVCpu->CTX_SUFF(pVM);
8790	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8791	if (rc == VINF_SUCCESS)
8792	{
8793	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8794	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8796	return VINF_SUCCESS;
8797	}
8798
8799	/* Forward #NM to the guest. */
8800	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8801	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8802	AssertRCReturn(rc, rc);
8803	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8804	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8805	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8806	return rc;
8807	}
8808
8809
8810	/**
8811	* VM-exit exception handler for #GP (General-protection exception).
8812	*
8813	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8814	*/
8815	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8816	{
8817	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8818	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8819
8820	int rc = VERR_INTERNAL_ERROR_5;
8821	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8822	{
8823	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8824	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8825	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8826	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8827	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8828	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8829	AssertRCReturn(rc, rc);
8830	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8831	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8832	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8833	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8834	return rc;
8835	#else
8836	/* We don't intercept #GP. */
8837	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8838	return VERR_VMX_UNEXPECTED_EXCEPTION;
8839	#endif
8840	}
8841
8842	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8843	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8844
8845	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8846	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8847	AssertRCReturn(rc, rc);
8848
8849	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8850	uint32_t cbOp = 0;
8851	PVM pVM = pVCpu->CTX_SUFF(pVM);
8852	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8853	if (RT_SUCCESS(rc))
8854	{
8855	rc = VINF_SUCCESS;
8856	Assert(cbOp == pDis->cbInstr);
8857	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8858	switch (pDis->pCurInstr->uOpcode)
8859	{
8860	case OP_CLI:
8861	pMixedCtx->eflags.Bits.u1IF = 0;
8862	pMixedCtx->rip += pDis->cbInstr;
8863	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8865	break;
8866
8867	case OP_STI:
8868	pMixedCtx->eflags.Bits.u1IF = 1;
8869	pMixedCtx->rip += pDis->cbInstr;
8870	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8871	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8872	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8873	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8874	break;
8875
8876	case OP_HLT:
8877	rc = VINF_EM_HALT;
8878	pMixedCtx->rip += pDis->cbInstr;
8879	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8881	break;
8882
8883	case OP_POPF:
8884	{
8885	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8886	uint32_t cbParm = 0;
8887	uint32_t uMask = 0;
8888	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8889	{
8890	cbParm = 4;
8891	uMask = 0xffffffff;
8892	}
8893	else
8894	{
8895	cbParm = 2;
8896	uMask = 0xffff;
8897	}
8898
8899	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8900	RTGCPTR GCPtrStack = 0;
8901	X86EFLAGS uEflags;
8902	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8903	&GCPtrStack);
8904	if (RT_SUCCESS(rc))
8905	{
8906	Assert(sizeof(uEflags.u32) >= cbParm);
8907	uEflags.u32 = 0;
8908	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8909	}
8910	if (RT_FAILURE(rc))
8911	{
8912	rc = VERR_EM_INTERPRETER;
8913	break;
8914	}
8915	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8916	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8917	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8918	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8919	pMixedCtx->eflags.Bits.u1RF = 0;
8920	pMixedCtx->esp += cbParm;
8921	pMixedCtx->esp &= uMask;
8922	pMixedCtx->rip += pDis->cbInstr;
8923	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8924	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8925	break;
8926	}
8927
8928	case OP_PUSHF:
8929	{
8930	uint32_t cbParm = 0;
8931	uint32_t uMask = 0;
8932	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8933	{
8934	cbParm = 4;
8935	uMask = 0xffffffff;
8936	}
8937	else
8938	{
8939	cbParm = 2;
8940	uMask = 0xffff;
8941	}
8942
8943	/* Get the stack pointer & push the contents of eflags onto the stack. */
8944	RTGCPTR GCPtrStack = 0;
8945	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8946	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8947	if (RT_FAILURE(rc))
8948	{
8949	rc = VERR_EM_INTERPRETER;
8950	break;
8951	}
8952	X86EFLAGS uEflags;
8953	uEflags = pMixedCtx->eflags;
8954	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8955	uEflags.Bits.u1RF = 0;
8956	uEflags.Bits.u1VM = 0;
8957
8958	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8959	if (RT_FAILURE(rc))
8960	{
8961	rc = VERR_EM_INTERPRETER;
8962	break;
8963	}
8964	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8965	pMixedCtx->esp -= cbParm;
8966	pMixedCtx->esp &= uMask;
8967	pMixedCtx->rip += pDis->cbInstr;
8968	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8970	break;
8971	}
8972
8973	case OP_IRET:
8974	{
8975	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8976	* instruction reference. */
8977	RTGCPTR GCPtrStack = 0;
8978	uint32_t uMask = 0xffff;
8979	uint16_t aIretFrame[3];
8980	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8981	{
8982	rc = VERR_EM_INTERPRETER;
8983	break;
8984	}
8985	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8986	&GCPtrStack);
8987	if (RT_SUCCESS(rc))
8988	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8989	if (RT_FAILURE(rc))
8990	{
8991	rc = VERR_EM_INTERPRETER;
8992	break;
8993	}
8994	pMixedCtx->eip = 0;
8995	pMixedCtx->ip = aIretFrame[0];
8996	pMixedCtx->cs.Sel = aIretFrame[1];
8997	pMixedCtx->cs.ValidSel = aIretFrame[1];
8998	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8999	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9000	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9001	pMixedCtx->sp += sizeof(aIretFrame);
9002	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9003	\| HM_CHANGED_GUEST_RFLAGS;
9004	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9005	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9006	break;
9007	}
9008
9009	case OP_INT:
9010	{
9011	uint16_t uVector = pDis->Param1.uValue & 0xff;
9012	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9013	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9014	break;
9015	}
9016
9017	case OP_INTO:
9018	{
9019	if (pMixedCtx->eflags.Bits.u1OF)
9020	{
9021	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9022	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9023	}
9024	break;
9025	}
9026
9027	default:
9028	{
9029	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9030	EMCODETYPE_SUPERVISOR);
9031	rc = VBOXSTRICTRC_VAL(rc2);
9032	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9033	Log4(("#GP rc=%Rrc\n", rc));
9034	break;
9035	}
9036	}
9037	}
9038	else
9039	rc = VERR_EM_INTERPRETER;
9040
9041	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9042	("#GP Unexpected rc=%Rrc\n", rc));
9043	return rc;
9044	}
9045
9046
9047	/**
9048	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9049	* the exception reported in the VMX transient structure back into the VM.
9050	*
9051	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9052	* up-to-date.
9053	*/
9054	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9055	{
9056	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9057
9058	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9059	hmR0VmxCheckExitDueToEventDelivery(). */
9060	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9061	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9062	AssertRCReturn(rc, rc);
9063	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9064	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9065	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9066	return VINF_SUCCESS;
9067	}
9068
9069
9070	/**
9071	* VM-exit exception handler for #PF (Page-fault exception).
9072	*/
9073	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9074	{
9075	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9076	PVM pVM = pVCpu->CTX_SUFF(pVM);
9077	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9078	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9079	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9080	AssertRCReturn(rc, rc);
9081
9082	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9083	if (pVM->hm.s.fNestedPaging)
9084	{
9085	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9086	{
9087	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9088	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9089	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9090	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9092	}
9093	else
9094	{
9095	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9096	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
9097	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9098	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9099	}
9100	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9101	return rc;
9102	}
9103	#else
9104	Assert(!pVM->hm.s.fNestedPaging);
9105	#endif
9106
9107	#ifdef VBOX_HM_WITH_GUEST_PATCHING
9108	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9109	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9110	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9111	AssertRCReturn(rc, rc);
9112	/* Shortcut for APIC TPR access, only for 32-bit guests. */
9113	if ( pVM->hm.s.fTRPPatchingAllowed
9114	&& pVM->hm.s.pGuestPatchMem
9115	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
9116	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
9117	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
9118	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
9119	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
9120	{
9121	RTGCPHYS GCPhys;
9122	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
9123	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9124	if ( rc == VINF_SUCCESS
9125	&& GCPhys == GCPhysApicBase)
9126	{
9127	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9128	AssertRCReturn(rc, rc);
9129
9130	/* Only attempt to patch the instruction once. */
9131	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9132	if (!pPatch)
9133	return VINF_EM_HM_PATCH_TPR_INSTR;
9134	}
9135	}
9136	#endif
9137
9138	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9139	AssertRCReturn(rc, rc);
9140
9141	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9142	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9143
9144	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9145	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9146	(RTGCPTR)pVmxTransient->uExitQualification);
9147
9148	Log4(("#PF: rc=%Rrc\n", rc));
9149	if (rc == VINF_SUCCESS)
9150	{
9151	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9152	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9153	* memory? We don't update the whole state here... */
9154	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9155	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9156	TRPMResetTrap(pVCpu);
9157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9158	return rc;
9159	}
9160	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9161	{
9162	if (!pVmxTransient->fVectoringPF)
9163	{
9164	/* It's a guest page fault and needs to be reflected to the guest. */
9165	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9166	TRPMResetTrap(pVCpu);
9167	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9168	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9169	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9170	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9171	}
9172	else
9173	{
9174	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9175	TRPMResetTrap(pVCpu);
9176	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9177	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9178	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9179	}
9180
9181	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9182	return VINF_SUCCESS;
9183	}
9184
9185	TRPMResetTrap(pVCpu);
9186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9187	return rc;
9188	}
9189

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46393

Download in other formats: