HMVMXR0.cpp@ 47105

Last change on this file since 47105 was 47105, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Added a todo.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 389.3 KB

Line
1	/* $Id: HMVMXR0.cpp 47105 2013-07-11 17:10:40Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#if defined(RT_ARCH_AMD64)
49	# define HMVMX_IS_64BIT_HOST_MODE() (true)
50	typedef RTHCUINTREG HMVMXHCUINTREG;
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	typedef uint64_t HMVMXHCUINTREG;
55	#else
56	# define HMVMX_IS_64BIT_HOST_MODE() (false)
57	typedef RTHCUINTREG HMVMXHCUINTREG;
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* @name Exception bitmap mask for real-mode guests (real-on-v86).
129	*
130	* We need to intercept all exceptions manually (except #PF). #NM is also
131	* handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be
132	* intercepted even in real-mode if we have Nested Paging support.
133	*/
134	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
135	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
136	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
137	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
138	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
139	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
140	\| RT_BIT(X86_XCPT_XF))
141	/** @} */
142
143	/**
144	* @name Exception bitmap mask for all contributory exceptions.
145	*
146	* Page fault is deliberately excluded here as it's conditional as to whether
147	* it's contributory or benign. Page faults are handled separately.
148	*/
149	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
150	\| RT_BIT(X86_XCPT_DE))
151	/** @} */
152
153	/** Maximum VM-instruction error number. */
154	#define HMVMX_INSTR_ERROR_MAX 28
155
156	/** Profiling macro. */
157	#ifdef HM_PROFILE_EXIT_DISPATCH
158	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
159	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
160	#else
161	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
162	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
163	#endif
164
165
166	/*******************************************************************************
167	* Structures and Typedefs *
168	*******************************************************************************/
169	/** @name VMX transient.
170	*
171	* A state structure for holding miscellaneous information across
172	* VMX non-root operation and restored after the transition.
173	*
174	* @{ */
175	typedef struct VMXTRANSIENT
176	{
177	/** The host's rflags/eflags. */
178	RTCCUINTREG uEFlags;
179	#if HC_ARCH_BITS == 32
180	uint32_t u32Alignment0;
181	#endif
182	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
183	uint64_t u64LStarMsr;
184	/** The guest's TPR value used for TPR shadowing. */
185	uint8_t u8GuestTpr;
186	/** Alignment. */
187	uint8_t abAlignment0[7];
188
189	/** The basic VM-exit reason. */
190	uint16_t uExitReason;
191	/** Alignment. */
192	uint16_t u16Alignment0;
193	/** The VM-exit interruption error code. */
194	uint32_t uExitIntrErrorCode;
195	/** The VM-exit exit qualification. */
196	uint64_t uExitQualification;
197
198	/** The VM-exit interruption-information field. */
199	uint32_t uExitIntrInfo;
200	/** The VM-exit instruction-length field. */
201	uint32_t cbInstr;
202	/** Whether the VM-entry failed or not. */
203	bool fVMEntryFailed;
204	/** Alignment. */
205	uint8_t abAlignment1[7];
206
207	/** The VM-entry interruption-information field. */
208	uint32_t uEntryIntrInfo;
209	/** The VM-entry exception error code field. */
210	uint32_t uEntryXcptErrorCode;
211	/** The VM-entry instruction length field. */
212	uint32_t cbEntryInstr;
213
214	/** IDT-vectoring information field. */
215	uint32_t uIdtVectoringInfo;
216	/** IDT-vectoring error code. */
217	uint32_t uIdtVectoringErrorCode;
218
219	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
220	uint32_t fVmcsFieldsRead;
221	/** Whether TSC-offsetting should be setup before VM-entry. */
222	bool fUpdateTscOffsettingAndPreemptTimer;
223	/** Whether the VM-exit was caused by a page-fault during delivery of a
224	* contributary exception or a page-fault. */
225	bool fVectoringPF;
226	} VMXTRANSIENT, *PVMXTRANSIENT;
227	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
228	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
229	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
230	/** @} */
231
232
233	/**
234	* MSR-bitmap read permissions.
235	*/
236	typedef enum VMXMSREXITREAD
237	{
238	/** Reading this MSR causes a VM-exit. */
239	VMXMSREXIT_INTERCEPT_READ = 0xb,
240	/** Reading this MSR does not cause a VM-exit. */
241	VMXMSREXIT_PASSTHRU_READ
242	} VMXMSREXITREAD;
243
244	/**
245	* MSR-bitmap write permissions.
246	*/
247	typedef enum VMXMSREXITWRITE
248	{
249	/** Writing to this MSR causes a VM-exit. */
250	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
251	/** Writing to this MSR does not cause a VM-exit. */
252	VMXMSREXIT_PASSTHRU_WRITE
253	} VMXMSREXITWRITE;
254
255
256	/*******************************************************************************
257	* Internal Functions *
258	*******************************************************************************/
259	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
260	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
261	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
262	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
263	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
264	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
265	#endif
266	#ifndef HMVMX_USE_FUNCTION_TABLE
267	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
268	#define HMVMX_EXIT_DECL static int
269	#else
270	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
271	#endif
272
273	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317
318	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
319	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
320	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
321	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
322	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
323	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
324	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
325
326
327	/*******************************************************************************
328	* Global Variables *
329	*******************************************************************************/
330	#ifdef HMVMX_USE_FUNCTION_TABLE
331	/**
332	* VM-exit handler.
333	*
334	* @returns VBox status code.
335	* @param pVCpu Pointer to the VMCPU.
336	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
337	* out-of-sync. Make sure to update the required
338	* fields before using them.
339	* @param pVmxTransient Pointer to the VMX-transient structure.
340	*/
341	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342	/** Pointer to VM-exit handler. */
343	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
344
345	/**
346	* VMX_EXIT dispatch table.
347	*/
348	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
349	{
350	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
351	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
352	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
353	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
354	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
355	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
356	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
357	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
358	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
359	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
360	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
361	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
362	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
363	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
364	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
365	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
366	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
367	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
368	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
369	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
370	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
371	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
372	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
373	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
374	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
375	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
376	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
377	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
378	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
379	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
380	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
381	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
382	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
383	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
384	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
385	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
387	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
388	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
389	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
390	/* 40 UNDEFINED */ hmR0VmxExitPause,
391	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
392	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
393	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
394	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
395	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
396	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
397	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
398	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
399	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
400	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
401	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
402	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
403	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
404	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
405	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
406	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
407	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
408	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
409	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
410	};
411	#endif /* HMVMX_USE_FUNCTION_TABLE */
412
413	#ifdef VBOX_STRICT
414	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
415	{
416	/* 0 */ "(Not Used)",
417	/* 1 */ "VMCALL executed in VMX root operation.",
418	/* 2 */ "VMCLEAR with invalid physical address.",
419	/* 3 */ "VMCLEAR with VMXON pointer.",
420	/* 4 */ "VMLAUNCH with non-clear VMCS.",
421	/* 5 */ "VMRESUME with non-launched VMCS.",
422	/* 6 */ "VMRESUME after VMXOFF",
423	/* 7 */ "VM entry with invalid control fields.",
424	/* 8 */ "VM entry with invalid host state fields.",
425	/* 9 */ "VMPTRLD with invalid physical address.",
426	/* 10 */ "VMPTRLD with VMXON pointer.",
427	/* 11 */ "VMPTRLD with incorrect revision identifier.",
428	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
429	/* 13 */ "VMWRITE to read-only VMCS component.",
430	/* 14 */ "(Not Used)",
431	/* 15 */ "VMXON executed in VMX root operation.",
432	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
433	/* 17 */ "VM entry with non-launched executing VMCS.",
434	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
435	/* 19 */ "VMCALL with non-clear VMCS.",
436	/* 20 */ "VMCALL with invalid VM-exit control fields.",
437	/* 21 */ "(Not Used)",
438	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
439	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
440	/* 24 */ "VMCALL with invalid SMM-monitor features.",
441	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
442	/* 26 */ "VM entry with events blocked by MOV SS.",
443	/* 27 */ "(Not Used)",
444	/* 28 */ "Invalid operand to INVEPT/INVVPID."
445	};
446	#endif /* VBOX_STRICT */
447
448
449
450	/**
451	* Updates the VM's last error record. If there was a VMX instruction error,
452	* reads the error data from the VMCS and updates VCPU's last error record as
453	* well.
454	*
455	* @param pVM Pointer to the VM.
456	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
457	* VERR_VMX_UNABLE_TO_START_VM or
458	* VERR_VMX_INVALID_VMCS_FIELD).
459	* @param rc The error code.
460	*/
461	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
462	{
463	AssertPtr(pVM);
464	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
465	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
466	{
467	AssertPtrReturnVoid(pVCpu);
468	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
469	}
470	pVM->hm.s.lLastError = rc;
471	}
472
473
474	/**
475	* Reads the VM-entry interruption-information field from the VMCS into the VMX
476	* transient structure.
477	*
478	* @returns VBox status code.
479	* @param pVmxTransient Pointer to the VMX transient structure.
480	*
481	* @remarks No-long-jump zone!!!
482	*/
483	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
484	{
485	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
486	AssertRCReturn(rc, rc);
487	return VINF_SUCCESS;
488	}
489
490
491	/**
492	* Reads the VM-entry exception error code field from the VMCS into
493	* the VMX transient structure.
494	*
495	* @returns VBox status code.
496	* @param pVmxTransient Pointer to the VMX transient structure.
497	*
498	* @remarks No-long-jump zone!!!
499	*/
500	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
501	{
502	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
503	AssertRCReturn(rc, rc);
504	return VINF_SUCCESS;
505	}
506
507
508	/**
509	* Reads the VM-entry exception error code field from the VMCS into
510	* the VMX transient structure.
511	*
512	* @returns VBox status code.
513	* @param pVCpu Pointer to the VMCPU.
514	* @param pVmxTransient Pointer to the VMX transient structure.
515	*
516	* @remarks No-long-jump zone!!!
517	*/
518	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
519	{
520	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
521	AssertRCReturn(rc, rc);
522	return VINF_SUCCESS;
523	}
524
525
526	/**
527	* Reads the VM-exit interruption-information field from the VMCS into the VMX
528	* transient structure.
529	*
530	* @returns VBox status code.
531	* @param pVCpu Pointer to the VMCPU.
532	* @param pVmxTransient Pointer to the VMX transient structure.
533	*/
534	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
535	{
536	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
537	{
538	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
539	AssertRCReturn(rc, rc);
540	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
541	}
542	return VINF_SUCCESS;
543	}
544
545
546	/**
547	* Reads the VM-exit interruption error code from the VMCS into the VMX
548	* transient structure.
549	*
550	* @returns VBox status code.
551	* @param pVCpu Pointer to the VMCPU.
552	* @param pVmxTransient Pointer to the VMX transient structure.
553	*/
554	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
555	{
556	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
557	{
558	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
559	AssertRCReturn(rc, rc);
560	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
561	}
562	return VINF_SUCCESS;
563	}
564
565
566	/**
567	* Reads the VM-exit instruction length field from the VMCS into the VMX
568	* transient structure.
569	*
570	* @returns VBox status code.
571	* @param pVCpu Pointer to the VMCPU.
572	* @param pVmxTransient Pointer to the VMX transient structure.
573	*/
574	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
575	{
576	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
577	{
578	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
579	AssertRCReturn(rc, rc);
580	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
581	}
582	return VINF_SUCCESS;
583	}
584
585
586	/**
587	* Reads the exit qualification from the VMCS into the VMX transient structure.
588	*
589	* @returns VBox status code.
590	* @param pVCpu Pointer to the VMCPU.
591	* @param pVmxTransient Pointer to the VMX transient structure.
592	*/
593	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
594	{
595	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
596	{
597	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
598	AssertRCReturn(rc, rc);
599	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
600	}
601	return VINF_SUCCESS;
602	}
603
604
605	/**
606	* Reads the IDT-vectoring information field from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*
612	* @remarks No-long-jump zone!!!
613	*/
614	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
615	{
616	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
617	{
618	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
619	AssertRCReturn(rc, rc);
620	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
621	}
622	return VINF_SUCCESS;
623	}
624
625
626	/**
627	* Reads the IDT-vectoring error code from the VMCS into the VMX
628	* transient structure.
629	*
630	* @returns VBox status code.
631	* @param pVmxTransient Pointer to the VMX transient structure.
632	*/
633	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
634	{
635	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
636	{
637	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
638	AssertRCReturn(rc, rc);
639	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
640	}
641	return VINF_SUCCESS;
642	}
643
644
645	/**
646	* Enters VMX root mode operation on the current CPU.
647	*
648	* @returns VBox status code.
649	* @param pVM Pointer to the VM (optional, can be NULL, after
650	* a resume).
651	* @param HCPhysCpuPage Physical address of the VMXON region.
652	* @param pvCpuPage Pointer to the VMXON region.
653	*/
654	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
655	{
656	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
657	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
658	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
659
660	if (pVM)
661	{
662	/* Write the VMCS revision dword to the VMXON region. */
663	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
664	}
665
666	/* Enable the VMX bit in CR4 if necessary. */
667	RTCCUINTREG uCr4 = ASMGetCR4();
668	if (!(uCr4 & X86_CR4_VMXE))
669	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
670
671	/* Enter VMX root mode. */
672	int rc = VMXEnable(HCPhysCpuPage);
673	if (RT_FAILURE(rc))
674	ASMSetCR4(uCr4);
675
676	return rc;
677	}
678
679
680	/**
681	* Exits VMX root mode operation on the current CPU.
682	*
683	* @returns VBox status code.
684	*/
685	static int hmR0VmxLeaveRootMode(void)
686	{
687	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
688
689	/* If we're for some reason not in VMX root mode, then don't leave it. */
690	RTCCUINTREG uHostCR4 = ASMGetCR4();
691	if (uHostCR4 & X86_CR4_VMXE)
692	{
693	/* Exit VMX root mode and clear the VMX bit in CR4. */
694	VMXDisable();
695	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
696	return VINF_SUCCESS;
697	}
698
699	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
700	}
701
702
703	/**
704	* Allocates and maps one physically contiguous page. The allocated page is
705	* zero'd out. (Used by various VT-x structures).
706	*
707	* @returns IPRT status code.
708	* @param pMemObj Pointer to the ring-0 memory object.
709	* @param ppVirt Where to store the virtual address of the
710	* allocation.
711	* @param pPhys Where to store the physical address of the
712	* allocation.
713	*/
714	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
715	{
716	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
717	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
718	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
719
720	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
721	if (RT_FAILURE(rc))
722	return rc;
723	ppVirt = RTR0MemObjAddress(pMemObj);
724	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
725	ASMMemZero32(*ppVirt, PAGE_SIZE);
726	return VINF_SUCCESS;
727	}
728
729
730	/**
731	* Frees and unmaps an allocated physical page.
732	*
733	* @param pMemObj Pointer to the ring-0 memory object.
734	* @param ppVirt Where to re-initialize the virtual address of
735	* allocation as 0.
736	* @param pHCPhys Where to re-initialize the physical address of the
737	* allocation as 0.
738	*/
739	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
740	{
741	AssertPtr(pMemObj);
742	AssertPtr(ppVirt);
743	AssertPtr(pHCPhys);
744	if (*pMemObj != NIL_RTR0MEMOBJ)
745	{
746	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
747	AssertRC(rc);
748	*pMemObj = NIL_RTR0MEMOBJ;
749	*ppVirt = 0;
750	*pHCPhys = 0;
751	}
752	}
753
754
755	/**
756	* Worker function to free VT-x related structures.
757	*
758	* @returns IPRT status code.
759	* @param pVM Pointer to the VM.
760	*/
761	static void hmR0VmxStructsFree(PVM pVM)
762	{
763	for (VMCPUID i = 0; i < pVM->cCpus; i++)
764	{
765	PVMCPU pVCpu = &pVM->aCpus[i];
766	AssertPtr(pVCpu);
767
768	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
769	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
770	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
771	#endif
772
773	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
774	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
775
776	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
777	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
778	}
779
780	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
781	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
782	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
783	#endif
784	}
785
786
787	/**
788	* Worker function to allocate VT-x related VM structures.
789	*
790	* @returns IPRT status code.
791	* @param pVM Pointer to the VM.
792	*/
793	static int hmR0VmxStructsAlloc(PVM pVM)
794	{
795	/*
796	* Initialize members up-front so we can cleanup properly on allocation failure.
797	*/
798	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
799	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
800	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
801	pVM->hm.s.vmx.HCPhys##a_Name = 0;
802
803	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
804	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
805	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
806	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
807
808	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
809	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
810	#endif
811	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
812
813	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
814	for (VMCPUID i = 0; i < pVM->cCpus; i++)
815	{
816	PVMCPU pVCpu = &pVM->aCpus[i];
817	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
818	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
819	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
820	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
821	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
822	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
823	#endif
824	}
825	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
826	#undef VMXLOCAL_INIT_VM_MEMOBJ
827
828	/*
829	* Allocate all the VT-x structures.
830	*/
831	int rc = VINF_SUCCESS;
832	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
834	if (RT_FAILURE(rc))
835	goto cleanup;
836	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
837	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
838	#endif
839
840	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
841	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
842	{
843	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
844	&pVM->hm.s.vmx.HCPhysApicAccess);
845	if (RT_FAILURE(rc))
846	goto cleanup;
847	}
848
849	/*
850	* Initialize per-VCPU VT-x structures.
851	*/
852	for (VMCPUID i = 0; i < pVM->cCpus; i++)
853	{
854	PVMCPU pVCpu = &pVM->aCpus[i];
855	AssertPtr(pVCpu);
856
857	/* Allocate the VM control structure (VMCS). */
858	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
859	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
860	if (RT_FAILURE(rc))
861	goto cleanup;
862
863	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
864	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
865	{
866	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
867	&pVCpu->hm.s.vmx.HCPhysVirtApic);
868	if (RT_FAILURE(rc))
869	goto cleanup;
870	}
871
872	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
873	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	{
875	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
876	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
877	if (RT_FAILURE(rc))
878	goto cleanup;
879	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
880	}
881
882	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
883	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
884	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
885	if (RT_FAILURE(rc))
886	goto cleanup;
887
888	/* Allocate the VM-exit MSR-load page for the host MSRs. */
889	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
890	if (RT_FAILURE(rc))
891	goto cleanup;
892	#endif
893	}
894
895	return VINF_SUCCESS;
896
897	cleanup:
898	hmR0VmxStructsFree(pVM);
899	return rc;
900	}
901
902
903	/**
904	* Does global VT-x initialization (called during module initialization).
905	*
906	* @returns VBox status code.
907	*/
908	VMMR0DECL(int) VMXR0GlobalInit(void)
909	{
910	#ifdef HMVMX_USE_FUNCTION_TABLE
911	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
912	# ifdef VBOX_STRICT
913	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
914	Assert(g_apfnVMExitHandlers[i]);
915	# endif
916	#endif
917	return VINF_SUCCESS;
918	}
919
920
921	/**
922	* Does global VT-x termination (called during module termination).
923	*/
924	VMMR0DECL(void) VMXR0GlobalTerm()
925	{
926	/* Nothing to do currently. */
927	}
928
929
930	/**
931	* Sets up and activates VT-x on the current CPU.
932	*
933	* @returns VBox status code.
934	* @param pCpu Pointer to the global CPU info struct.
935	* @param pVM Pointer to the VM (can be NULL after a host resume
936	* operation).
937	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
938	* fEnabledByHost is true).
939	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
940	* @a fEnabledByHost is true).
941	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
942	* enable VT-x on the host.
943	*/
944	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
945	{
946	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
947	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
948
949	if (!fEnabledByHost)
950	{
951	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
952	if (RT_FAILURE(rc))
953	return rc;
954	}
955
956	/*
957	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
958	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
959	* each time while reusing a VPID after hitting the MaxASID limit once.
960	*/
961	if ( pVM
962	&& pVM->hm.s.fNestedPaging)
963	{
964	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
965	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
966	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
967	pCpu->fFlushAsidBeforeUse = false;
968	}
969	else
970	{
971	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
972	* without Nested Paging triggered this function) we still have the risk
973	* of potentially running with stale TLB-entries from other hypervisors
974	* when later we use a VM with NestedPaging. To fix this properly we will
975	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
976	* 'vmx_ept_vpid_caps' from it. Sigh. */
977	pCpu->fFlushAsidBeforeUse = true;
978	}
979
980	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
981	++pCpu->cTlbFlushes;
982
983	return VINF_SUCCESS;
984	}
985
986
987	/**
988	* Deactivates VT-x on the current CPU.
989	*
990	* @returns VBox status code.
991	* @param pCpu Pointer to the global CPU info struct.
992	* @param pvCpuPage Pointer to the VMXON region.
993	* @param HCPhysCpuPage Physical address of the VMXON region.
994	*
995	* @remarks This function should never be called when SUPR0EnableVTx() or
996	* similar was used to enable VT-x on the host.
997	*/
998	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
999	{
1000	NOREF(pCpu);
1001	NOREF(pvCpuPage);
1002	NOREF(HCPhysCpuPage);
1003
1004	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1005	return hmR0VmxLeaveRootMode();
1006	}
1007
1008
1009	/**
1010	* Sets the permission bits for the specified MSR in the MSR bitmap.
1011	*
1012	* @param pVCpu Pointer to the VMCPU.
1013	* @param uMSR The MSR value.
1014	* @param enmRead Whether reading this MSR causes a VM-exit.
1015	* @param enmWrite Whether writing this MSR causes a VM-exit.
1016	*/
1017	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1018	{
1019	int32_t iBit;
1020	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1021
1022	/*
1023	* Layout:
1024	* 0x000 - 0x3ff - Low MSR read bits
1025	* 0x400 - 0x7ff - High MSR read bits
1026	* 0x800 - 0xbff - Low MSR write bits
1027	* 0xc00 - 0xfff - High MSR write bits
1028	*/
1029	if (uMsr <= 0x00001FFF)
1030	iBit = uMsr;
1031	else if ( uMsr >= 0xC0000000
1032	&& uMsr <= 0xC0001FFF)
1033	{
1034	iBit = (uMsr - 0xC0000000);
1035	pbMsrBitmap += 0x400;
1036	}
1037	else
1038	{
1039	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1040	return;
1041	}
1042
1043	Assert(iBit <= 0x1fff);
1044	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1045	ASMBitSet(pbMsrBitmap, iBit);
1046	else
1047	ASMBitClear(pbMsrBitmap, iBit);
1048
1049	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1050	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1051	else
1052	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1053	}
1054
1055
1056	/**
1057	* Flushes the TLB using EPT.
1058	*
1059	* @returns VBox status code.
1060	* @param pVM Pointer to the VM.
1061	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1062	* enmFlush).
1063	* @param enmFlush Type of flush.
1064	*/
1065	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1066	{
1067	AssertPtr(pVM);
1068	Assert(pVM->hm.s.fNestedPaging);
1069
1070	uint64_t descriptor[2];
1071	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1072	descriptor[0] = 0;
1073	else
1074	{
1075	Assert(pVCpu);
1076	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1077	}
1078	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1079
1080	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1081	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1082	rc));
1083	if ( RT_SUCCESS(rc)
1084	&& pVCpu)
1085	{
1086	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1087	}
1088	}
1089
1090
1091	/**
1092	* Flushes the TLB using VPID.
1093	*
1094	* @returns VBox status code.
1095	* @param pVM Pointer to the VM.
1096	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1097	* enmFlush).
1098	* @param enmFlush Type of flush.
1099	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1100	* on @a enmFlush).
1101	*/
1102	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1103	{
1104	AssertPtr(pVM);
1105	Assert(pVM->hm.s.vmx.fVpid);
1106
1107	uint64_t descriptor[2];
1108	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1109	{
1110	descriptor[0] = 0;
1111	descriptor[1] = 0;
1112	}
1113	else
1114	{
1115	AssertPtr(pVCpu);
1116	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1117	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1118	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1119	descriptor[1] = GCPtr;
1120	}
1121
1122	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1123	AssertMsg(rc == VINF_SUCCESS,
1124	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1125	if ( RT_SUCCESS(rc)
1126	&& pVCpu)
1127	{
1128	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1129	}
1130	}
1131
1132
1133	/**
1134	* Invalidates a guest page by guest virtual address. Only relevant for
1135	* EPT/VPID, otherwise there is nothing really to invalidate.
1136	*
1137	* @returns VBox status code.
1138	* @param pVM Pointer to the VM.
1139	* @param pVCpu Pointer to the VMCPU.
1140	* @param GCVirt Guest virtual address of the page to invalidate.
1141	*/
1142	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1143	{
1144	AssertPtr(pVM);
1145	AssertPtr(pVCpu);
1146	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1147
1148	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1149	if (!fFlushPending)
1150	{
1151	/*
1152	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1153	* See @bugref{6043} and @bugref{6177}.
1154	*
1155	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1156	* function maybe called in a loop with individual addresses.
1157	*/
1158	if (pVM->hm.s.vmx.fVpid)
1159	{
1160	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1161	{
1162	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1163	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1164	}
1165	else
1166	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1167	}
1168	else if (pVM->hm.s.fNestedPaging)
1169	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1170	}
1171
1172	return VINF_SUCCESS;
1173	}
1174
1175
1176	/**
1177	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1178	* otherwise there is nothing really to invalidate.
1179	*
1180	* @returns VBox status code.
1181	* @param pVM Pointer to the VM.
1182	* @param pVCpu Pointer to the VMCPU.
1183	* @param GCPhys Guest physical address of the page to invalidate.
1184	*/
1185	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1186	{
1187	LogFlowFunc(("%RGp\n", GCPhys));
1188
1189	/*
1190	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1191	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1192	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1193	*/
1194	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1195	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1196	return VINF_SUCCESS;
1197	}
1198
1199
1200	/**
1201	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1202	* case where neither EPT nor VPID is supported by the CPU.
1203	*
1204	* @param pVM Pointer to the VM.
1205	* @param pVCpu Pointer to the VMCPU.
1206	*
1207	* @remarks Called with interrupts disabled.
1208	*/
1209	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1210	{
1211	NOREF(pVM);
1212	AssertPtr(pVCpu);
1213	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1214	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1215
1216	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1217	AssertPtr(pCpu);
1218
1219	pVCpu->hm.s.TlbShootdown.cPages = 0;
1220	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1221	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1222	pVCpu->hm.s.fForceTLBFlush = false;
1223	return;
1224	}
1225
1226
1227	/**
1228	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1229	*
1230	* @param pVM Pointer to the VM.
1231	* @param pVCpu Pointer to the VMCPU.
1232	* @remarks All references to "ASID" in this function pertains to "VPID" in
1233	* Intel's nomenclature. The reason is, to avoid confusion in compare
1234	* statements since the host-CPU copies are named "ASID".
1235	*
1236	* @remarks Called with interrupts disabled.
1237	*/
1238	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1239	{
1240	#ifdef VBOX_WITH_STATISTICS
1241	bool fTlbFlushed = false;
1242	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1243	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1244	if (!fTlbFlushed) \
1245	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1246	} while (0)
1247	#else
1248	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1249	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1250	#endif
1251
1252	AssertPtr(pVM);
1253	AssertPtr(pVCpu);
1254	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1255	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1256	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1257
1258	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1259	AssertPtr(pCpu);
1260
1261	/*
1262	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1263	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1264	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1265	*/
1266	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1267	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1268	{
1269	++pCpu->uCurrentAsid;
1270	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1271	{
1272	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1273	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1274	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1275	}
1276
1277	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1278	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1279	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1280
1281	/*
1282	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1283	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1284	*/
1285	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1286	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1287	HMVMX_SET_TAGGED_TLB_FLUSHED();
1288	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1289	}
1290
1291	/* Check for explicit TLB shootdowns. */
1292	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1293	{
1294	/*
1295	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1296	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1297	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1298	* but not guest-physical mappings.
1299	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1300	*/
1301	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1302	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1303	HMVMX_SET_TAGGED_TLB_FLUSHED();
1304	}
1305
1306	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1307	* not be executed. See hmQueueInvlPage() where it is commented
1308	* out. Support individual entry flushing someday. */
1309	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1310	{
1311	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1312
1313	/*
1314	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1315	* as supported by the CPU.
1316	*/
1317	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1318	{
1319	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1320	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1321	}
1322	else
1323	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1324
1325	HMVMX_SET_TAGGED_TLB_FLUSHED();
1326	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1327	}
1328
1329	pVCpu->hm.s.TlbShootdown.cPages = 0;
1330	pVCpu->hm.s.fForceTLBFlush = false;
1331
1332	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1333
1334	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1335	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1336	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1337	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1338	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1339	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1340	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1341	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1342
1343	/* Update VMCS with the VPID. */
1344	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1345	AssertRC(rc);
1346
1347	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1348	}
1349
1350
1351	/**
1352	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1353	*
1354	* @returns VBox status code.
1355	* @param pVM Pointer to the VM.
1356	* @param pVCpu Pointer to the VMCPU.
1357	*
1358	* @remarks Called with interrupts disabled.
1359	*/
1360	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1361	{
1362	AssertPtr(pVM);
1363	AssertPtr(pVCpu);
1364	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1365	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1366
1367	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1368	AssertPtr(pCpu);
1369
1370	/*
1371	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1372	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1373	*/
1374	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1375	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1376	{
1377	pVCpu->hm.s.fForceTLBFlush = true;
1378	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1379	}
1380
1381	/* Check for explicit TLB shootdown flushes. */
1382	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1383	{
1384	pVCpu->hm.s.fForceTLBFlush = true;
1385	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1386	}
1387
1388	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1389	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1390
1391	if (pVCpu->hm.s.fForceTLBFlush)
1392	{
1393	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1394	pVCpu->hm.s.fForceTLBFlush = false;
1395	}
1396	else
1397	{
1398	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1399	* not be executed. See hmQueueInvlPage() where it is commented
1400	* out. Support individual entry flushing someday. */
1401	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1402	{
1403	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1404	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1405	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1406	}
1407	else
1408	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1409	}
1410
1411	pVCpu->hm.s.TlbShootdown.cPages = 0;
1412	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1413	}
1414
1415
1416	/**
1417	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1418	*
1419	* @returns VBox status code.
1420	* @param pVM Pointer to the VM.
1421	* @param pVCpu Pointer to the VMCPU.
1422	*
1423	* @remarks Called with interrupts disabled.
1424	*/
1425	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1426	{
1427	AssertPtr(pVM);
1428	AssertPtr(pVCpu);
1429	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1430	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1431
1432	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1433
1434	/*
1435	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1436	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1437	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1438	*/
1439	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1440	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1441	{
1442	pVCpu->hm.s.fForceTLBFlush = true;
1443	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1444	}
1445
1446	/* Check for explicit TLB shootdown flushes. */
1447	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1448	{
1449	/*
1450	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1451	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1452	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1453	*/
1454	pVCpu->hm.s.fForceTLBFlush = true;
1455	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1456	}
1457
1458	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1459	if (pVCpu->hm.s.fForceTLBFlush)
1460	{
1461	++pCpu->uCurrentAsid;
1462	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1463	{
1464	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1465	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1466	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1467	}
1468
1469	pVCpu->hm.s.fForceTLBFlush = false;
1470	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1471	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1472	if (pCpu->fFlushAsidBeforeUse)
1473	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1474	}
1475	else
1476	{
1477	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1478	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1479	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1480	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1481
1482	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1483	* not be executed. See hmQueueInvlPage() where it is commented
1484	* out. Support individual entry flushing someday. */
1485	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1486	{
1487	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1488	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1489	{
1490	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1491	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1492	}
1493	else
1494	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1495	}
1496	else
1497	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1498	}
1499
1500	pVCpu->hm.s.TlbShootdown.cPages = 0;
1501	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1502
1503	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1504	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1505	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1506	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1507	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1508	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1509
1510	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1511	AssertRC(rc);
1512	}
1513
1514
1515	/**
1516	* Flushes the guest TLB entry based on CPU capabilities.
1517	*
1518	* @param pVCpu Pointer to the VMCPU.
1519	*/
1520	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1521	{
1522	PVM pVM = pVCpu->CTX_SUFF(pVM);
1523	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1524	{
1525	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1526	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1527	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1528	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1529	default:
1530	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1531	break;
1532	}
1533	}
1534
1535
1536	/**
1537	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1538	* TLB entries from the host TLB before VM-entry.
1539	*
1540	* @returns VBox status code.
1541	* @param pVM Pointer to the VM.
1542	*/
1543	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1544	{
1545	/*
1546	* Determine optimal flush type for Nested Paging.
1547	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1548	* guest execution (see hmR3InitFinalizeR0()).
1549	*/
1550	if (pVM->hm.s.fNestedPaging)
1551	{
1552	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1553	{
1554	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1555	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1556	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1557	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1558	else
1559	{
1560	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1561	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1562	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1563	}
1564
1565	/* Make sure the write-back cacheable memory type for EPT is supported. */
1566	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1567	{
1568	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1569	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1570	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1571	}
1572	}
1573	else
1574	{
1575	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1576	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1577	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1578	}
1579	}
1580
1581	/*
1582	* Determine optimal flush type for VPID.
1583	*/
1584	if (pVM->hm.s.vmx.fVpid)
1585	{
1586	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1587	{
1588	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1589	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1590	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1591	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1592	else
1593	{
1594	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1595	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1596	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1597	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1598	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1599	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1600	pVM->hm.s.vmx.fVpid = false;
1601	}
1602	}
1603	else
1604	{
1605	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1606	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1607	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1608	pVM->hm.s.vmx.fVpid = false;
1609	}
1610	}
1611
1612	/*
1613	* Setup the handler for flushing tagged-TLBs.
1614	*/
1615	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1616	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1617	else if (pVM->hm.s.fNestedPaging)
1618	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1619	else if (pVM->hm.s.vmx.fVpid)
1620	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1621	else
1622	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1623	return VINF_SUCCESS;
1624	}
1625
1626
1627	/**
1628	* Sets up pin-based VM-execution controls in the VMCS.
1629	*
1630	* @returns VBox status code.
1631	* @param pVM Pointer to the VM.
1632	* @param pVCpu Pointer to the VMCPU.
1633	*/
1634	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1635	{
1636	AssertPtr(pVM);
1637	AssertPtr(pVCpu);
1638
1639	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1640	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1641
1642	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1643	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1644	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1645
1646	/* Enable the VMX preemption timer. */
1647	if (pVM->hm.s.vmx.fUsePreemptTimer)
1648	{
1649	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1650	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1651	}
1652
1653	if ((val & zap) != val)
1654	{
1655	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1656	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1657	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1658	}
1659
1660	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1661	AssertRCReturn(rc, rc);
1662
1663	/* Update VCPU with the currently set pin-based VM-execution controls. */
1664	pVCpu->hm.s.vmx.u32PinCtls = val;
1665	return rc;
1666	}
1667
1668
1669	/**
1670	* Sets up processor-based VM-execution controls in the VMCS.
1671	*
1672	* @returns VBox status code.
1673	* @param pVM Pointer to the VM.
1674	* @param pVMCPU Pointer to the VMCPU.
1675	*/
1676	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1677	{
1678	AssertPtr(pVM);
1679	AssertPtr(pVCpu);
1680
1681	int rc = VERR_INTERNAL_ERROR_5;
1682	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1683	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1684
1685	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1686	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1687	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1688	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1689	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1690	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1691	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1692
1693	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1694	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1695	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1696	{
1697	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1698	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1699	}
1700
1701	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1702	if (!pVM->hm.s.fNestedPaging)
1703	{
1704	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1705	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1706	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1707	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1708	}
1709
1710	/* Use TPR shadowing if supported by the CPU. */
1711	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1712	{
1713	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1714	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1715	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1716	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1717	AssertRCReturn(rc, rc);
1718
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1720	/* CR8 writes causes a VM-exit based on TPR threshold. */
1721	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1722	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1723	}
1724	else
1725	{
1726	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1727	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1728	}
1729
1730	/* Use MSR-bitmaps if supported by the CPU. */
1731	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1732	{
1733	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1734
1735	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1736	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1737	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1738	AssertRCReturn(rc, rc);
1739
1740	/*
1741	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1742	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1743	*/
1744	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1745	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1746	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1747	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1748	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1749	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1750	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1751	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1752	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1753	}
1754
1755	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1756	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1757	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1758
1759	if ((val & zap) != val)
1760	{
1761	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1762	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1763	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1764	}
1765
1766	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1767	AssertRCReturn(rc, rc);
1768
1769	/* Update VCPU with the currently set processor-based VM-execution controls. */
1770	pVCpu->hm.s.vmx.u32ProcCtls = val;
1771
1772	/*
1773	* Secondary processor-based VM-execution controls.
1774	*/
1775	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1776	{
1777	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1778	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1779
1780	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1781	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1782
1783	if (pVM->hm.s.fNestedPaging)
1784	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1785	else
1786	{
1787	/*
1788	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1789	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1790	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1791	*/
1792	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1793	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1794	}
1795
1796	if (pVM->hm.s.vmx.fVpid)
1797	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1798
1799	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1800	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1801
1802	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1803	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1804	* done dynamically. */
1805	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1806	{
1807	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1808	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1809	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1810	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1811	AssertRCReturn(rc, rc);
1812	}
1813
1814	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1815	{
1816	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1817	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1818	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1819	}
1820
1821	if ((val & zap) != val)
1822	{
1823	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1824	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1825	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1826	}
1827
1828	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1829	AssertRCReturn(rc, rc);
1830
1831	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1832	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1833	}
1834
1835	return VINF_SUCCESS;
1836	}
1837
1838
1839	/**
1840	* Sets up miscellaneous (everything other than Pin & Processor-based
1841	* VM-execution) control fields in the VMCS.
1842	*
1843	* @returns VBox status code.
1844	* @param pVM Pointer to the VM.
1845	* @param pVCpu Pointer to the VMCPU.
1846	*/
1847	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1848	{
1849	AssertPtr(pVM);
1850	AssertPtr(pVCpu);
1851
1852	int rc = VERR_GENERAL_FAILURE;
1853
1854	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1855	#if 0
1856	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1857	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1858	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1859
1860	/*
1861	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1862	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1863	* We thus use the exception bitmap to control it rather than use both.
1864	*/
1865	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1866	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1867
1868	/** @todo Explore possibility of using IO-bitmaps. */
1869	/* All IO & IOIO instructions cause VM-exits. */
1870	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1871	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1872
1873	/* Initialize the MSR-bitmap area. */
1874	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1875	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1876	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1877	#endif
1878
1879	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1880	/* Setup MSR autoloading/storing. */
1881	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1882	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1883	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1884	AssertRCReturn(rc, rc);
1885	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1886	AssertRCReturn(rc, rc);
1887
1888	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1889	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1890	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1891	AssertRCReturn(rc, rc);
1892	#endif
1893
1894	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1895	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1896	AssertRCReturn(rc, rc);
1897
1898	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1899	#if 0
1900	/* Setup debug controls */
1901	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1902	AssertRCReturn(rc, rc);
1903	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1904	AssertRCReturn(rc, rc);
1905	#endif
1906
1907	return rc;
1908	}
1909
1910
1911	/**
1912	* Sets up the initial exception bitmap in the VMCS based on static conditions
1913	* (i.e. conditions that cannot ever change at runtime).
1914	*
1915	* @returns VBox status code.
1916	* @param pVM Pointer to the VM.
1917	* @param pVCpu Pointer to the VMCPU.
1918	*/
1919	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1920	{
1921	AssertPtr(pVM);
1922	AssertPtr(pVCpu);
1923
1924	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1925
1926	uint32_t u32XcptBitmap = 0;
1927
1928	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1929	if (!pVM->hm.s.fNestedPaging)
1930	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1931
1932	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1933	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1934	AssertRCReturn(rc, rc);
1935	return rc;
1936	}
1937
1938
1939	/**
1940	* Sets up the initial guest-state mask. The guest-state mask is consulted
1941	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1942	* for the nested virtualization case (as it would cause a VM-exit).
1943	*
1944	* @param pVCpu Pointer to the VMCPU.
1945	*/
1946	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1947	{
1948	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1949	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1950	return VINF_SUCCESS;
1951	}
1952
1953
1954	/**
1955	* Does per-VM VT-x initialization.
1956	*
1957	* @returns VBox status code.
1958	* @param pVM Pointer to the VM.
1959	*/
1960	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1961	{
1962	LogFlowFunc(("pVM=%p\n", pVM));
1963
1964	int rc = hmR0VmxStructsAlloc(pVM);
1965	if (RT_FAILURE(rc))
1966	{
1967	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1968	return rc;
1969	}
1970
1971	return VINF_SUCCESS;
1972	}
1973
1974
1975	/**
1976	* Does per-VM VT-x termination.
1977	*
1978	* @returns VBox status code.
1979	* @param pVM Pointer to the VM.
1980	*/
1981	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1982	{
1983	LogFlowFunc(("pVM=%p\n", pVM));
1984
1985	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1986	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1987	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1988	#endif
1989	hmR0VmxStructsFree(pVM);
1990	return VINF_SUCCESS;
1991	}
1992
1993
1994	/**
1995	* Sets up the VM for execution under VT-x.
1996	* This function is only called once per-VM during initalization.
1997	*
1998	* @returns VBox status code.
1999	* @param pVM Pointer to the VM.
2000	*/
2001	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2002	{
2003	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2004	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2005
2006	LogFlowFunc(("pVM=%p\n", pVM));
2007
2008	/*
2009	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2010	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2011	*/
2012	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2013	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2014	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2015	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2016	{
2017	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2018	return VERR_INTERNAL_ERROR;
2019	}
2020
2021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2022	/*
2023	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2024	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2025	*/
2026	if ( pVM->hm.s.fAllow64BitGuests
2027	&& !HMVMX_IS_64BIT_HOST_MODE())
2028	{
2029	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2030	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2031	}
2032	#endif
2033
2034	/* Initialize these always, see hmR3InitFinalizeR0().*/
2035	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2036	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2037
2038	/* Setup the tagged-TLB flush handlers. */
2039	int rc = hmR0VmxSetupTaggedTlb(pVM);
2040	if (RT_FAILURE(rc))
2041	{
2042	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2043	return rc;
2044	}
2045
2046	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2047	{
2048	PVMCPU pVCpu = &pVM->aCpus[i];
2049	AssertPtr(pVCpu);
2050	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2051
2052	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2053	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2054
2055	/* Set revision dword at the beginning of the VMCS structure. */
2056	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2057
2058	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2059	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2060	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2061	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2062
2063	/* Load this VMCS as the current VMCS. */
2064	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2065	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2066	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2067
2068	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2069	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2070	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2071
2072	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2073	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2074	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2075
2076	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2077	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2078	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2079
2080	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2081	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2082	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2083
2084	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2085	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2086	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2087
2088	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2089	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2090	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2091	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2092	#endif
2093
2094	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2095	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2096	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2097	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2098
2099	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2100	}
2101
2102	return VINF_SUCCESS;
2103	}
2104
2105
2106	/**
2107	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2108	* the VMCS.
2109	*
2110	* @returns VBox status code.
2111	* @param pVM Pointer to the VM.
2112	* @param pVCpu Pointer to the VMCPU.
2113	*/
2114	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2115	{
2116	RTCCUINTREG uReg = ASMGetCR0();
2117	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2118	AssertRCReturn(rc, rc);
2119
2120	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2121	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2122	if (HMVMX_IS_64BIT_HOST_MODE())
2123	{
2124	uint64_t uRegCR3 = HMR0Get64bitCR3();
2125	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2126	}
2127	else
2128	#endif
2129	{
2130	uReg = ASMGetCR3();
2131	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2132	}
2133	AssertRCReturn(rc, rc);
2134
2135	uReg = ASMGetCR4();
2136	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2137	AssertRCReturn(rc, rc);
2138	return rc;
2139	}
2140
2141
2142	/**
2143	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2144	* the host-state area in the VMCS.
2145	*
2146	* @returns VBox status code.
2147	* @param pVM Pointer to the VM.
2148	* @param pVCpu Pointer to the VMCPU.
2149	*/
2150	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2151	{
2152	int rc = VERR_INTERNAL_ERROR_5;
2153	RTSEL uSelDS = 0;
2154	RTSEL uSelES = 0;
2155	RTSEL uSelFS = 0;
2156	RTSEL uSelGS = 0;
2157	RTSEL uSelTR = 0;
2158
2159	/*
2160	* Host DS, ES, FS and GS segment registers.
2161	*/
2162	#if HC_ARCH_BITS == 64
2163	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2164	uSelDS = ASMGetDS();
2165	uSelES = ASMGetES();
2166	uSelFS = ASMGetFS();
2167	uSelGS = ASMGetGS();
2168	#endif
2169
2170	/*
2171	* Host CS and SS segment registers.
2172	*/
2173	RTSEL uSelCS;
2174	RTSEL uSelSS;
2175	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2176	if (HMVMX_IS_64BIT_HOST_MODE())
2177	{
2178	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2179	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2180	}
2181	else
2182	{
2183	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2184	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2185	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2186	}
2187	#else
2188	uSelCS = ASMGetCS();
2189	uSelSS = ASMGetSS();
2190	#endif
2191
2192	/*
2193	* Host TR segment register.
2194	*/
2195	uSelTR = ASMGetTR();
2196
2197	#if HC_ARCH_BITS == 64
2198	/*
2199	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2200	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2201	*/
2202	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2203	{
2204	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2205	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2206	uSelDS = 0;
2207	}
2208	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2209	{
2210	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2211	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2212	uSelES = 0;
2213	}
2214	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2215	{
2216	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2217	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2218	uSelFS = 0;
2219	}
2220	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2221	{
2222	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2223	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2224	uSelGS = 0;
2225	}
2226	#endif
2227
2228	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2229	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2230	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2231	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2232	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2233	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2234	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2235	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2236	Assert(uSelCS);
2237	Assert(uSelTR);
2238
2239	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2240	#if 0
2241	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2242	Assert(uSelSS != 0);
2243	#endif
2244
2245	/* Write these host selector fields into the host-state area in the VMCS. */
2246	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2247	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2248	#if HC_ARCH_BITS == 64
2249	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2250	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2251	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2252	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2253	#endif
2254	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2255
2256	/*
2257	* Host GDTR and IDTR.
2258	*/
2259	RTGDTR Gdtr;
2260	RT_ZERO(Gdtr);
2261	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2262	if (HMVMX_IS_64BIT_HOST_MODE())
2263	{
2264	X86XDTR64 Gdtr64;
2265	X86XDTR64 Idtr64;
2266	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2267	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2268	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2269
2270	Gdtr.cbGdt = Gdtr64.cb;
2271	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2272	}
2273	else
2274	#endif
2275	{
2276	RTIDTR Idtr;
2277	ASMGetGDTR(&Gdtr);
2278	ASMGetIDTR(&Idtr);
2279	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2280	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2281
2282	#if HC_ARCH_BITS == 64
2283	/*
2284	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2285	* maximum limit (0xffff) on every VM-exit.
2286	*/
2287	if (Gdtr.cbGdt != 0xffff)
2288	{
2289	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2290	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2291	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2292	}
2293
2294	/*
2295	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2296	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2297	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2298	*/
2299	if (Idtr.cbIdt < 0x0fff)
2300	{
2301	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2302	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2303	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2304	}
2305	#endif
2306	}
2307
2308	/*
2309	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2310	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2311	*/
2312	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2313	{
2314	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2315	return VERR_VMX_INVALID_HOST_STATE;
2316	}
2317
2318	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2319	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2320	if (HMVMX_IS_64BIT_HOST_MODE())
2321	{
2322	/* We need the 64-bit TR base for hybrid darwin. */
2323	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2324	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2325	}
2326	else
2327	#endif
2328	{
2329	uintptr_t uTRBase;
2330	#if HC_ARCH_BITS == 64
2331	uTRBase = X86DESC64_BASE(pDesc);
2332
2333	/*
2334	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2335	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2336	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2337	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2338	*
2339	* [1] See Intel spec. 3.5 "System Descriptor Types".
2340	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2341	*/
2342	Assert(pDesc->System.u4Type == 11);
2343	if ( pDesc->System.u16LimitLow != 0x67
2344	\|\| pDesc->System.u4LimitHigh)
2345	{
2346	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2347	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2348
2349	/* Store the GDTR here as we need it while restoring TR. */
2350	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2351	}
2352	#else
2353	uTRBase = X86DESC_BASE(pDesc);
2354	#endif
2355	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2356	}
2357	AssertRCReturn(rc, rc);
2358
2359	/*
2360	* Host FS base and GS base.
2361	*/
2362	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2363	if (HMVMX_IS_64BIT_HOST_MODE())
2364	{
2365	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2366	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2367	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2368	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2369
2370	# if HC_ARCH_BITS == 64
2371	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2372	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2373	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2374	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2375	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2376	# endif
2377	}
2378	#endif
2379	return rc;
2380	}
2381
2382
2383	/**
2384	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2385	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2386	* the host after every successful VM exit.
2387	*
2388	* @returns VBox status code.
2389	* @param pVM Pointer to the VM.
2390	* @param pVCpu Pointer to the VMCPU.
2391	*/
2392	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2393	{
2394	AssertPtr(pVCpu);
2395	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2396
2397	int rc = VINF_SUCCESS;
2398	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2399	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2400	uint32_t cHostMsrs = 0;
2401	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2402
2403	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2404	{
2405	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2406
2407	# if HC_ARCH_BITS == 64
2408	/* Paranoia. 64-bit code requires these bits to be set always. */
2409	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2410
2411	/*
2412	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2413	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2414	* some reason (e.g. allow transparent reads) we would activate the code below.
2415	*/
2416	# if 0
2417	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2418	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2419	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2420	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2421	if (CPUMIsGuestInLongMode(pVCpu))
2422	{
2423	uint64_t u64GuestEfer;
2424	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2425	AssertRC(rc);
2426
2427	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2428	{
2429	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2430	pHostMsr->u32Reserved = 0;
2431	pHostMsr->u64Value = u64HostEfer;
2432	pHostMsr++; cHostMsrs++;
2433	}
2434	}
2435	# endif
2436	# else /* HC_ARCH_BITS != 64 */
2437	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2438	pHostMsr->u32Reserved = 0;
2439	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2440	if (CPUMIsGuestInLongMode(pVCpu))
2441	{
2442	/* Must match the EFER value in our 64 bits switcher. */
2443	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2444	}
2445	else
2446	# endif
2447	pHostMsr->u64Value = u64HostEfer;
2448	pHostMsr++; cHostMsrs++;
2449	# endif /* HC_ARCH_BITS == 64 */
2450	}
2451
2452	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2453	if (HMVMX_IS_64BIT_HOST_MODE())
2454	{
2455	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2456	pHostMsr->u32Reserved = 0;
2457	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2458	pHostMsr++; cHostMsrs++;
2459	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2460	pHostMsr->u32Reserved = 0;
2461	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2462	pHostMsr++; cHostMsrs++;
2463	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2464	pHostMsr->u32Reserved = 0;
2465	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2466	pHostMsr++; cHostMsrs++;
2467	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2468	pHostMsr->u32Reserved = 0;
2469	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2470	pHostMsr++; cHostMsrs++;
2471	}
2472	# endif
2473
2474	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2475	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2476	{
2477	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2478	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2479	}
2480
2481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2482	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2483
2484	/*
2485	* Host Sysenter MSRs.
2486	*/
2487	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2488	AssertRCReturn(rc, rc);
2489	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2490	if (HMVMX_IS_64BIT_HOST_MODE())
2491	{
2492	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2493	AssertRCReturn(rc, rc);
2494	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2495	}
2496	else
2497	{
2498	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2499	AssertRCReturn(rc, rc);
2500	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2501	}
2502	#elif HC_ARCH_BITS == 32
2503	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2504	AssertRCReturn(rc, rc);
2505	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2506	#else
2507	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2508	AssertRCReturn(rc, rc);
2509	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2510	#endif
2511	AssertRCReturn(rc, rc);
2512
2513	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2514	* hmR0VmxSetupExitCtls() !! */
2515	return rc;
2516	}
2517
2518
2519	/**
2520	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2521	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2522	* controls".
2523	*
2524	* @returns VBox status code.
2525	* @param pVCpu Pointer to the VMCPU.
2526	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2527	* out-of-sync. Make sure to update the required fields
2528	* before using them.
2529	*
2530	* @remarks No-long-jump zone!!!
2531	*/
2532	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2533	{
2534	int rc = VINF_SUCCESS;
2535	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2536	{
2537	PVM pVM = pVCpu->CTX_SUFF(pVM);
2538	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2539	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2540
2541	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2542	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2543
2544	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2545	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2546	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2547	else
2548	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2549
2550	/*
2551	* The following should not be set (since we're not in SMM mode):
2552	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2553	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2554	*/
2555
2556	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2557	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2558	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2559
2560	if ((val & zap) != val)
2561	{
2562	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2563	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2564	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2565	}
2566
2567	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2568	AssertRCReturn(rc, rc);
2569
2570	/* Update VCPU with the currently set VM-exit controls. */
2571	pVCpu->hm.s.vmx.u32EntryCtls = val;
2572	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2573	}
2574	return rc;
2575	}
2576
2577
2578	/**
2579	* Sets up the VM-exit controls in the VMCS.
2580	*
2581	* @returns VBox status code.
2582	* @param pVM Pointer to the VM.
2583	* @param pVCpu Pointer to the VMCPU.
2584	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2585	* out-of-sync. Make sure to update the required fields
2586	* before using them.
2587	*
2588	* @remarks requires EFER.
2589	*/
2590	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2591	{
2592	int rc = VINF_SUCCESS;
2593	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2594	{
2595	PVM pVM = pVCpu->CTX_SUFF(pVM);
2596	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2597	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2598
2599	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2600	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2601
2602	/*
2603	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2604	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2605	*/
2606	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2607	if (HMVMX_IS_64BIT_HOST_MODE())
2608	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2609	else
2610	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2611	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2612	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2613	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2614	else
2615	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2616	#endif
2617
2618	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2619	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2620
2621	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2622	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2623	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2624	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2625	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2626
2627	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2628	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2629
2630	if ((val & zap) != val)
2631	{
2632	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2633	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2634	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2635	}
2636
2637	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2638	AssertRCReturn(rc, rc);
2639
2640	/* Update VCPU with the currently set VM-exit controls. */
2641	pVCpu->hm.s.vmx.u32ExitCtls = val;
2642	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2643	}
2644	return rc;
2645	}
2646
2647
2648	/**
2649	* Loads the guest APIC and related state.
2650	*
2651	* @returns VBox status code.
2652	* @param pVM Pointer to the VM.
2653	* @param pVCpu Pointer to the VMCPU.
2654	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2655	* out-of-sync. Make sure to update the required fields
2656	* before using them.
2657	*/
2658	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2659	{
2660	int rc = VINF_SUCCESS;
2661	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2662	{
2663	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2664	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2665	{
2666	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2667
2668	bool fPendingIntr = false;
2669	uint8_t u8Tpr = 0;
2670	uint8_t u8PendingIntr = 0;
2671	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2672	AssertRCReturn(rc, rc);
2673
2674	/*
2675	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2676	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2677	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2678	* the interrupt when we VM-exit for other reasons.
2679	*/
2680	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2681	uint32_t u32TprThreshold = 0;
2682	if (fPendingIntr)
2683	{
2684	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2685	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2686	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2687	if (u8PendingPriority <= u8TprPriority)
2688	u32TprThreshold = u8PendingPriority;
2689	else
2690	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2691	}
2692	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2693
2694	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2695	AssertRCReturn(rc, rc);
2696	}
2697
2698	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2699	}
2700	return rc;
2701	}
2702
2703
2704	/**
2705	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2706	*
2707	* @returns Guest's interruptibility-state.
2708	* @param pVCpu Pointer to the VMCPU.
2709	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2710	* out-of-sync. Make sure to update the required fields
2711	* before using them.
2712	*
2713	* @remarks No-long-jump zone!!!
2714	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2715	*/
2716	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2717	{
2718	/*
2719	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2720	* inhibit interrupts or clear any existing interrupt-inhibition.
2721	*/
2722	uint32_t uIntrState = 0;
2723	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2724	{
2725	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2726	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2727	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2728	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2729	{
2730	/*
2731	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2732	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2733	*/
2734	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2735	}
2736	else if (pMixedCtx->eflags.Bits.u1IF)
2737	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2738	else
2739	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2740	}
2741	return uIntrState;
2742	}
2743
2744
2745	/**
2746	* Loads the guest's interruptibility-state into the guest-state area in the
2747	* VMCS.
2748	*
2749	* @returns VBox status code.
2750	* @param pVCpu Pointer to the VMCPU.
2751	* @param uIntrState The interruptibility-state to set.
2752	*/
2753	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2754	{
2755	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2756	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2757	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2758	AssertRCReturn(rc, rc);
2759	return rc;
2760	}
2761
2762
2763	/**
2764	* Loads the guest's RIP into the guest-state area in the VMCS.
2765	*
2766	* @returns VBox status code.
2767	* @param pVCpu Pointer to the VMCPU.
2768	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2769	* out-of-sync. Make sure to update the required fields
2770	* before using them.
2771	*
2772	* @remarks No-long-jump zone!!!
2773	*/
2774	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2775	{
2776	int rc = VINF_SUCCESS;
2777	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2778	{
2779	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2780	AssertRCReturn(rc, rc);
2781	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2782	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2783	}
2784	return rc;
2785	}
2786
2787
2788	/**
2789	* Loads the guest's RSP into the guest-state area in the VMCS.
2790	*
2791	* @returns VBox status code.
2792	* @param pVCpu Pointer to the VMCPU.
2793	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2794	* out-of-sync. Make sure to update the required fields
2795	* before using them.
2796	*
2797	* @remarks No-long-jump zone!!!
2798	*/
2799	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2800	{
2801	int rc = VINF_SUCCESS;
2802	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2803	{
2804	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2805	AssertRCReturn(rc, rc);
2806	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2807	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2808	}
2809	return rc;
2810	}
2811
2812
2813	/**
2814	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2815	*
2816	* @returns VBox status code.
2817	* @param pVCpu Pointer to the VMCPU.
2818	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2819	* out-of-sync. Make sure to update the required fields
2820	* before using them.
2821	*
2822	* @remarks No-long-jump zone!!!
2823	*/
2824	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2825	{
2826	int rc = VINF_SUCCESS;
2827	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2828	{
2829	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2830	Let us assert it as such and use 32-bit VMWRITE. */
2831	Assert(!(pMixedCtx->rflags.u64 >> 32));
2832	X86EFLAGS uEFlags = pMixedCtx->eflags;
2833	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2834	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2835
2836	/*
2837	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2838	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2839	*/
2840	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2841	{
2842	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2843	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2844	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2845	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2846	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2847	}
2848
2849	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2850	AssertRCReturn(rc, rc);
2851
2852	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2853	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2854	}
2855	return rc;
2856	}
2857
2858
2859	/**
2860	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2861	*
2862	* @returns VBox status code.
2863	* @param pVCpu Pointer to the VMCPU.
2864	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2865	* out-of-sync. Make sure to update the required fields
2866	* before using them.
2867	*
2868	* @remarks No-long-jump zone!!!
2869	*/
2870	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2871	{
2872	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2873	AssertRCReturn(rc, rc);
2874	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2875	AssertRCReturn(rc, rc);
2876	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2877	AssertRCReturn(rc, rc);
2878	return rc;
2879	}
2880
2881
2882	/**
2883	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2884	* in the VMCS.
2885	*
2886	* @returns VBox status code.
2887	* @param pVM Pointer to the VM.
2888	* @param pVCpu Pointer to the VMCPU.
2889	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2890	* out-of-sync. Make sure to update the required fields
2891	* before using them.
2892	*
2893	* @remarks No-long-jump zone!!!
2894	*/
2895	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2896	{
2897	int rc = VINF_SUCCESS;
2898	PVM pVM = pVCpu->CTX_SUFF(pVM);
2899
2900	/*
2901	* Guest CR0.
2902	* Guest FPU.
2903	*/
2904	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2905	{
2906	Assert(!(pCtx->cr0 >> 32));
2907	uint32_t u32GuestCR0 = pCtx->cr0;
2908
2909	/* The guest's view (read access) of its CR0 is unblemished. */
2910	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2911	AssertRCReturn(rc, rc);
2912	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2913
2914	/* Setup VT-x's view of the guest CR0. */
2915	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2916	if (pVM->hm.s.fNestedPaging)
2917	{
2918	if (CPUMIsGuestPagingEnabledEx(pCtx))
2919	{
2920	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2921	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2922	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2923	}
2924	else
2925	{
2926	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2927	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2928	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2929	}
2930
2931	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2932	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2933	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2934
2935	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2936	AssertRCReturn(rc, rc);
2937	}
2938	else
2939	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2940
2941	/*
2942	* Guest FPU bits.
2943	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2944	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2945	*/
2946	u32GuestCR0 \|= X86_CR0_NE;
2947	bool fInterceptNM = false;
2948	if (CPUMIsGuestFPUStateActive(pVCpu))
2949	{
2950	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2951	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2952	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2953	}
2954	else
2955	{
2956	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2957	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2958	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2959	}
2960
2961	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2962	bool fInterceptMF = false;
2963	if (!(pCtx->cr0 & X86_CR0_NE))
2964	fInterceptMF = true;
2965
2966	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2967	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2968	{
2969	Assert(PDMVmmDevHeapIsEnabled(pVM));
2970	Assert(pVM->hm.s.vmx.pRealModeTSS);
2971	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2972	fInterceptNM = true;
2973	fInterceptMF = true;
2974	}
2975	else
2976	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2977
2978	if (fInterceptNM)
2979	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2980	else
2981	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2982
2983	if (fInterceptMF)
2984	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2985	else
2986	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2987
2988	/* Additional intercepts for debugging, define these yourself explicitly. */
2989	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2990	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
2991	\| RT_BIT(X86_XCPT_BP)
2992	\| RT_BIT(X86_XCPT_DB)
2993	\| RT_BIT(X86_XCPT_DE)
2994	\| RT_BIT(X86_XCPT_NM)
2995	\| RT_BIT(X86_XCPT_UD)
2996	\| RT_BIT(X86_XCPT_NP)
2997	\| RT_BIT(X86_XCPT_SS)
2998	\| RT_BIT(X86_XCPT_GP)
2999	\| RT_BIT(X86_XCPT_PF)
3000	\| RT_BIT(X86_XCPT_MF)
3001	;
3002	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3003	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3004	#endif
3005
3006	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3007
3008	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3009	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3010	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3011	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3012	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3013	else
3014	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3015
3016	u32GuestCR0 \|= uSetCR0;
3017	u32GuestCR0 &= uZapCR0;
3018	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3019
3020	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3021	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3022	AssertRCReturn(rc, rc);
3023	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3024	AssertRCReturn(rc, rc);
3025	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3026
3027	/*
3028	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3029	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3030	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3031	*/
3032	uint32_t u32CR0Mask = 0;
3033	u32CR0Mask = X86_CR0_PE
3034	\| X86_CR0_NE
3035	\| X86_CR0_WP
3036	\| X86_CR0_PG
3037	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3038	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3039	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3040	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3041	u32CR0Mask &= ~X86_CR0_PE;
3042	if (pVM->hm.s.fNestedPaging)
3043	u32CR0Mask &= ~X86_CR0_WP;
3044
3045	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3046	if (fInterceptNM)
3047	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3048	else
3049	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3050
3051	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3052	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3053	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3054	AssertRCReturn(rc, rc);
3055
3056	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3057	}
3058
3059	/*
3060	* Guest CR2.
3061	* It's always loaded in the assembler code. Nothing to do here.
3062	*/
3063
3064	/*
3065	* Guest CR3.
3066	*/
3067	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3068	{
3069	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3070	if (pVM->hm.s.fNestedPaging)
3071	{
3072	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3073
3074	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3075	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3076	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3077	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3078
3079	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3080	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3081	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3082
3083	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3084	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3085	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3086	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3087
3088	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3089	AssertRCReturn(rc, rc);
3090	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3091
3092	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3093	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3094	{
3095	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3096	if (CPUMIsGuestInPAEModeEx(pCtx))
3097	{
3098	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3099	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3100	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3101	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3102	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3103	}
3104
3105	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3106	have Unrestricted Execution to handle the guest when it's not using paging. */
3107	GCPhysGuestCR3 = pCtx->cr3;
3108	}
3109	else
3110	{
3111	/*
3112	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3113	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3114	* EPT takes care of translating it to host-physical addresses.
3115	*/
3116	RTGCPHYS GCPhys;
3117	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3118	Assert(PDMVmmDevHeapIsEnabled(pVM));
3119
3120	/* We obtain it here every time as the guest could have relocated this PCI region. */
3121	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3122	AssertRCReturn(rc, rc);
3123
3124	GCPhysGuestCR3 = GCPhys;
3125	}
3126
3127	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3128	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3129	}
3130	else
3131	{
3132	/* Non-nested paging case, just use the hypervisor's CR3. */
3133	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3134
3135	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3136	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3137	}
3138	AssertRCReturn(rc, rc);
3139
3140	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3141	}
3142
3143	/*
3144	* Guest CR4.
3145	*/
3146	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3147	{
3148	Assert(!(pCtx->cr4 >> 32));
3149	uint32_t u32GuestCR4 = pCtx->cr4;
3150
3151	/* The guest's view of its CR4 is unblemished. */
3152	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3153	AssertRCReturn(rc, rc);
3154	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3155
3156	/* Setup VT-x's view of the guest CR4. */
3157	/*
3158	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3159	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3160	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3161	*/
3162	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3163	{
3164	Assert(pVM->hm.s.vmx.pRealModeTSS);
3165	Assert(PDMVmmDevHeapIsEnabled(pVM));
3166	u32GuestCR4 &= ~X86_CR4_VME;
3167	}
3168
3169	if (pVM->hm.s.fNestedPaging)
3170	{
3171	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3172	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3173	{
3174	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3175	u32GuestCR4 \|= X86_CR4_PSE;
3176	/* Our identity mapping is a 32 bits page directory. */
3177	u32GuestCR4 &= ~X86_CR4_PAE;
3178	}
3179	/* else use guest CR4.*/
3180	}
3181	else
3182	{
3183	/*
3184	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3185	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3186	*/
3187	switch (pVCpu->hm.s.enmShadowMode)
3188	{
3189	case PGMMODE_REAL: /* Real-mode. */
3190	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3191	case PGMMODE_32_BIT: /* 32-bit paging. */
3192	{
3193	u32GuestCR4 &= ~X86_CR4_PAE;
3194	break;
3195	}
3196
3197	case PGMMODE_PAE: /* PAE paging. */
3198	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3199	{
3200	u32GuestCR4 \|= X86_CR4_PAE;
3201	break;
3202	}
3203
3204	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3205	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3206	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3207	break;
3208	#endif
3209	default:
3210	AssertFailed();
3211	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3212	}
3213	}
3214
3215	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3216	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3217	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3218	u32GuestCR4 \|= uSetCR4;
3219	u32GuestCR4 &= uZapCR4;
3220
3221	/* Write VT-x's view of the guest CR4 into the VMCS. */
3222	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3223	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3224	AssertRCReturn(rc, rc);
3225
3226	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3227	uint32_t u32CR4Mask = 0;
3228	u32CR4Mask = X86_CR4_VME
3229	\| X86_CR4_PAE
3230	\| X86_CR4_PGE
3231	\| X86_CR4_PSE
3232	\| X86_CR4_VMXE;
3233	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3234	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3235	AssertRCReturn(rc, rc);
3236
3237	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3238	}
3239	return rc;
3240	}
3241
3242
3243	/**
3244	* Loads the guest debug registers into the guest-state area in the VMCS.
3245	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3246	*
3247	* @returns VBox status code.
3248	* @param pVCpu Pointer to the VMCPU.
3249	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3250	* out-of-sync. Make sure to update the required fields
3251	* before using them.
3252	*
3253	* @remarks No-long-jump zone!!!
3254	*/
3255	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3256	{
3257	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3258	return VINF_SUCCESS;
3259
3260	#ifdef VBOX_STRICT
3261	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3262	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3263	{
3264	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3265	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3266	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3267	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3268	}
3269	#endif
3270
3271	int rc = VERR_INTERNAL_ERROR_5;
3272	PVM pVM = pVCpu->CTX_SUFF(pVM);
3273	bool fInterceptDB = false;
3274	bool fInterceptMovDRx = false;
3275	if (DBGFIsStepping(pVCpu))
3276	{
3277	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3278	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3279	{
3280	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3281	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3282	AssertRCReturn(rc, rc);
3283	Assert(fInterceptDB == false);
3284	}
3285	else
3286	{
3287	fInterceptDB = true;
3288	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3289	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3290	}
3291	}
3292
3293	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3294	{
3295	if (!CPUMIsHyperDebugStateActive(pVCpu))
3296	{
3297	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3298	AssertRC(rc);
3299	}
3300	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3301	fInterceptMovDRx = true;
3302	}
3303	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3304	{
3305	if (!CPUMIsGuestDebugStateActive(pVCpu))
3306	{
3307	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3308	AssertRC(rc);
3309	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3310	}
3311	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3312	Assert(fInterceptMovDRx == false);
3313	}
3314	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3315	{
3316	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3317	fInterceptMovDRx = true;
3318	}
3319
3320	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3321	if (fInterceptDB)
3322	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3323	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3324	{
3325	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3326	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3327	#endif
3328	}
3329
3330	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3331	if (fInterceptMovDRx)
3332	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3333	else
3334	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3335
3336	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3337	AssertRCReturn(rc, rc);
3338	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3339	AssertRCReturn(rc, rc);
3340
3341	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3342	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3343	AssertRCReturn(rc, rc);
3344
3345	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3346	return rc;
3347	}
3348
3349
3350	#ifdef VBOX_STRICT
3351	/**
3352	* Strict function to validate segment registers.
3353	*
3354	* @remarks Requires CR0.
3355	*/
3356	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3357	{
3358	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3359	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3360	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3361	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3362	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3363	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3364	{
3365	/* Protected mode checks */
3366	/* CS */
3367	Assert(pCtx->cs.Attr.n.u1Present);
3368	Assert(!(pCtx->cs.Attr.u & 0xf00));
3369	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3370	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3371	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3372	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3373	\|\| (pCtx->cs.Attr.n.u1Granularity));
3374	/* CS cannot be loaded with NULL in protected mode. */
3375	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3376	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3377	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3378	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3379	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3380	else
3381	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3382	/* SS */
3383	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3384	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3385	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3386	if ( !(pCtx->cr0 & X86_CR0_PE)
3387	\|\| pCtx->cs.Attr.n.u4Type == 3)
3388	{
3389	Assert(!pCtx->ss.Attr.n.u2Dpl);
3390	}
3391	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3392	{
3393	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3394	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3395	Assert(pCtx->ss.Attr.n.u1Present);
3396	Assert(!(pCtx->ss.Attr.u & 0xf00));
3397	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3398	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3399	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3400	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3401	\|\| (pCtx->ss.Attr.n.u1Granularity));
3402	}
3403	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3404	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3405	{
3406	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3407	Assert(pCtx->ds.Attr.n.u1Present);
3408	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3409	Assert(!(pCtx->ds.Attr.u & 0xf00));
3410	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3411	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3412	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3413	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3414	\|\| (pCtx->ds.Attr.n.u1Granularity));
3415	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3416	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3417	}
3418	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3419	{
3420	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3421	Assert(pCtx->es.Attr.n.u1Present);
3422	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3423	Assert(!(pCtx->es.Attr.u & 0xf00));
3424	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3425	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3426	\|\| !(pCtx->es.Attr.n.u1Granularity));
3427	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3428	\|\| (pCtx->es.Attr.n.u1Granularity));
3429	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3430	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3431	}
3432	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3433	{
3434	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3435	Assert(pCtx->fs.Attr.n.u1Present);
3436	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3437	Assert(!(pCtx->fs.Attr.u & 0xf00));
3438	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3439	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3440	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3441	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3442	\|\| (pCtx->fs.Attr.n.u1Granularity));
3443	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3444	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3445	}
3446	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3447	{
3448	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3449	Assert(pCtx->gs.Attr.n.u1Present);
3450	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3451	Assert(!(pCtx->gs.Attr.u & 0xf00));
3452	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3453	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3454	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3455	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3456	\|\| (pCtx->gs.Attr.n.u1Granularity));
3457	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3458	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3459	}
3460	/* 64-bit capable CPUs. */
3461	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3462	Assert(!(pCtx->cs.u64Base >> 32));
3463	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3464	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3465	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3466	# endif
3467	}
3468	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3469	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3470	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3471	{
3472	/* Real and v86 mode checks. */
3473	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3474	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3475	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3476	{
3477	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3478	}
3479	else
3480	{
3481	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3482	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3483	}
3484
3485	/* CS */
3486	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3487	Assert(pCtx->cs.u32Limit == 0xffff);
3488	Assert(u32CSAttr == 0xf3);
3489	/* SS */
3490	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3491	Assert(pCtx->ss.u32Limit == 0xffff);
3492	Assert(u32SSAttr == 0xf3);
3493	/* DS */
3494	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3495	Assert(pCtx->ds.u32Limit == 0xffff);
3496	Assert(u32DSAttr == 0xf3);
3497	/* ES */
3498	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3499	Assert(pCtx->es.u32Limit == 0xffff);
3500	Assert(u32ESAttr == 0xf3);
3501	/* FS */
3502	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3503	Assert(pCtx->fs.u32Limit == 0xffff);
3504	Assert(u32FSAttr == 0xf3);
3505	/* GS */
3506	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3507	Assert(pCtx->gs.u32Limit == 0xffff);
3508	Assert(u32GSAttr == 0xf3);
3509	/* 64-bit capable CPUs. */
3510	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3511	Assert(!(pCtx->cs.u64Base >> 32));
3512	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3513	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3514	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3515	# endif
3516	}
3517	}
3518	#endif /* VBOX_STRICT */
3519
3520
3521	/**
3522	* Writes a guest segment register into the guest-state area in the VMCS.
3523	*
3524	* @returns VBox status code.
3525	* @param pVCpu Pointer to the VMCPU.
3526	* @param idxSel Index of the selector in the VMCS.
3527	* @param idxLimit Index of the segment limit in the VMCS.
3528	* @param idxBase Index of the segment base in the VMCS.
3529	* @param idxAccess Index of the access rights of the segment in the VMCS.
3530	* @param pSelReg Pointer to the segment selector.
3531	* @param pCtx Pointer to the guest-CPU context.
3532	*
3533	* @remarks No-long-jump zone!!!
3534	*/
3535	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3536	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3537	{
3538	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3539	AssertRCReturn(rc, rc);
3540	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3541	AssertRCReturn(rc, rc);
3542	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3543	AssertRCReturn(rc, rc);
3544
3545	uint32_t u32Access = pSelReg->Attr.u;
3546	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3547	{
3548	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3549	u32Access = 0xf3;
3550	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3551	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3552	}
3553	else
3554	{
3555	/*
3556	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3557	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3558	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3559	* loaded in protected-mode have their attribute as 0.
3560	*/
3561	if (!u32Access)
3562	u32Access = HMVMX_SEL_UNUSABLE;
3563	}
3564
3565	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3566	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3567	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3568
3569	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3570	AssertRCReturn(rc, rc);
3571	return rc;
3572	}
3573
3574
3575	/**
3576	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3577	* into the guest-state area in the VMCS.
3578	*
3579	* @returns VBox status code.
3580	* @param pVM Pointer to the VM.
3581	* @param pVCPU Pointer to the VMCPU.
3582	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3583	* out-of-sync. Make sure to update the required fields
3584	* before using them.
3585	*
3586	* @remarks Requires CR0 (strict builds validation).
3587	* @remarks No-long-jump zone!!!
3588	*/
3589	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3590	{
3591	int rc = VERR_INTERNAL_ERROR_5;
3592	PVM pVM = pVCpu->CTX_SUFF(pVM);
3593
3594	/*
3595	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3596	*/
3597	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3598	{
3599	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3600	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3601	{
3602	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3603	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3604	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3605	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3606	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3607	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3608	}
3609
3610	#ifdef VBOX_WITH_REM
3611	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3612	{
3613	Assert(pVM->hm.s.vmx.pRealModeTSS);
3614	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3615	if ( pVCpu->hm.s.vmx.fWasInRealMode
3616	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3617	{
3618	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3619	in real-mode (e.g. OpenBSD 4.0) */
3620	REMFlushTBs(pVM);
3621	Log4(("Load: Switch to protected mode detected!\n"));
3622	pVCpu->hm.s.vmx.fWasInRealMode = false;
3623	}
3624	}
3625	#endif
3626	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3627	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3628	AssertRCReturn(rc, rc);
3629	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3630	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3631	AssertRCReturn(rc, rc);
3632	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3633	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3634	AssertRCReturn(rc, rc);
3635	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3636	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3637	AssertRCReturn(rc, rc);
3638	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3639	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3640	AssertRCReturn(rc, rc);
3641	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3642	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3643	AssertRCReturn(rc, rc);
3644
3645	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3646	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3647	#ifdef VBOX_STRICT
3648	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3649	#endif
3650	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3651	}
3652
3653	/*
3654	* Guest TR.
3655	*/
3656	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3657	{
3658	/*
3659	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3660	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3661	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3662	*/
3663	uint16_t u16Sel = 0;
3664	uint32_t u32Limit = 0;
3665	uint64_t u64Base = 0;
3666	uint32_t u32AccessRights = 0;
3667
3668	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3669	{
3670	u16Sel = pMixedCtx->tr.Sel;
3671	u32Limit = pMixedCtx->tr.u32Limit;
3672	u64Base = pMixedCtx->tr.u64Base;
3673	u32AccessRights = pMixedCtx->tr.Attr.u;
3674	}
3675	else
3676	{
3677	Assert(pVM->hm.s.vmx.pRealModeTSS);
3678	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3679
3680	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3681	RTGCPHYS GCPhys;
3682	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3683	AssertRCReturn(rc, rc);
3684
3685	X86DESCATTR DescAttr;
3686	DescAttr.u = 0;
3687	DescAttr.n.u1Present = 1;
3688	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3689
3690	u16Sel = 0;
3691	u32Limit = HM_VTX_TSS_SIZE;
3692	u64Base = GCPhys; /* in real-mode phys = virt. */
3693	u32AccessRights = DescAttr.u;
3694	}
3695
3696	/* Validate. */
3697	Assert(!(u16Sel & RT_BIT(2)));
3698	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3699	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3700	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3701	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3702	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3703	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3704	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3705	Assert( (u32Limit & 0xfff) == 0xfff
3706	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3707	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3708	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3709
3710	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3711	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3712	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3713	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3714
3715	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3716	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3717	}
3718
3719	/*
3720	* Guest GDTR.
3721	*/
3722	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3723	{
3724	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3725	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3726
3727	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3728	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3729	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3730	}
3731
3732	/*
3733	* Guest LDTR.
3734	*/
3735	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3736	{
3737	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3738	uint32_t u32Access = 0;
3739	if (!pMixedCtx->ldtr.Attr.u)
3740	u32Access = HMVMX_SEL_UNUSABLE;
3741	else
3742	u32Access = pMixedCtx->ldtr.Attr.u;
3743
3744	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3745	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3746	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3747	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3748
3749	/* Validate. */
3750	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3751	{
3752	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3753	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3754	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3755	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3756	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3757	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3758	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3759	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3760	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3761	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3762	}
3763
3764	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3765	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3766	}
3767
3768	/*
3769	* Guest IDTR.
3770	*/
3771	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3772	{
3773	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3774	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3775
3776	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3777	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3778	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3779	}
3780
3781	return VINF_SUCCESS;
3782	}
3783
3784
3785	/**
3786	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3787	* areas. These MSRs will automatically be loaded to the host CPU on every
3788	* successful VM entry and stored from the host CPU on every successful VM exit.
3789	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3790	*
3791	* @returns VBox status code.
3792	* @param pVCpu Pointer to the VMCPU.
3793	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3794	* out-of-sync. Make sure to update the required fields
3795	* before using them.
3796	*
3797	* @remarks No-long-jump zone!!!
3798	*/
3799	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3800	{
3801	AssertPtr(pVCpu);
3802	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3803
3804	/*
3805	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3806	*/
3807	int rc = VINF_SUCCESS;
3808	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3809	{
3810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3811	PVM pVM = pVCpu->CTX_SUFF(pVM);
3812	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3813	uint32_t cGuestMsrs = 0;
3814
3815	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3816	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3817	* when the guest really is in 64-bit mode. */
3818	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3819	if (fSupportsLongMode)
3820	{
3821	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3822	pGuestMsr->u32Reserved = 0;
3823	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3824	pGuestMsr++; cGuestMsrs++;
3825	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3826	pGuestMsr->u32Reserved = 0;
3827	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3828	pGuestMsr++; cGuestMsrs++;
3829	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3830	pGuestMsr->u32Reserved = 0;
3831	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3832	pGuestMsr++; cGuestMsrs++;
3833	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3834	pGuestMsr->u32Reserved = 0;
3835	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3836	pGuestMsr++; cGuestMsrs++;
3837	}
3838
3839	/*
3840	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3841	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3842	*/
3843	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3844	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3845	{
3846	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3847	pGuestMsr->u32Reserved = 0;
3848	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3849	AssertRCReturn(rc, rc);
3850	pGuestMsr++; cGuestMsrs++;
3851	}
3852
3853	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3854	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3855	{
3856	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3857	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3858	}
3859
3860	/* Update the VCPU's copy of the guest MSR count. */
3861	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3862	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3863	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3864	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3865
3866	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3867	}
3868
3869	/*
3870	* Guest Sysenter MSRs.
3871	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3872	* VM-exits on WRMSRs for these MSRs.
3873	*/
3874	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3875	{
3876	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3877	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3878	}
3879	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3880	{
3881	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3882	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3883	}
3884	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3885	{
3886	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3887	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3888	}
3889
3890	return rc;
3891	}
3892
3893
3894	/**
3895	* Loads the guest activity state into the guest-state area in the VMCS.
3896	*
3897	* @returns VBox status code.
3898	* @param pVCpu Pointer to the VMCPU.
3899	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3900	* out-of-sync. Make sure to update the required fields
3901	* before using them.
3902	*
3903	* @remarks No-long-jump zone!!!
3904	*/
3905	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3906	{
3907	/** @todo See if we can make use of other states, e.g.
3908	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3909	int rc = VINF_SUCCESS;
3910	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3911	{
3912	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3913	AssertRCReturn(rc, rc);
3914	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3915	}
3916	return rc;
3917	}
3918
3919
3920	/**
3921	* Sets up the appropriate function to run guest code.
3922	*
3923	* @returns VBox status code.
3924	* @param pVCpu Pointer to the VMCPU.
3925	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3926	* out-of-sync. Make sure to update the required fields
3927	* before using them.
3928	*
3929	* @remarks No-long-jump zone!!!
3930	*/
3931	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3932	{
3933	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3934	{
3935	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3936	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3937	#endif
3938	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3939	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3940	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3941	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3942	#else
3943	/* 64-bit host or hybrid host. */
3944	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3945	#endif
3946	}
3947	else
3948	{
3949	/* Guest is not in long mode, use the 32-bit handler. */
3950	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3951	}
3952	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3953	return VINF_SUCCESS;
3954	}
3955
3956
3957	/**
3958	* Wrapper for running the guest code in VT-x.
3959	*
3960	* @returns VBox strict status code.
3961	* @param pVM Pointer to the VM.
3962	* @param pVCpu Pointer to the VMCPU.
3963	* @param pCtx Pointer to the guest-CPU context.
3964	*
3965	* @remarks No-long-jump zone!!!
3966	*/
3967	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3968	{
3969	/*
3970	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3971	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3972	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3973	*/
3974	#ifdef VBOX_WITH_KERNEL_USING_XMM
3975	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3976	#else
3977	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3978	#endif
3979	}
3980
3981
3982	/**
3983	* Reports world-switch error and dumps some useful debug info.
3984	*
3985	* @param pVM Pointer to the VM.
3986	* @param pVCpu Pointer to the VMCPU.
3987	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3988	* @param pCtx Pointer to the guest-CPU context.
3989	* @param pVmxTransient Pointer to the VMX transient structure (only
3990	* exitReason updated).
3991	*/
3992	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3993	{
3994	Assert(pVM);
3995	Assert(pVCpu);
3996	Assert(pCtx);
3997	Assert(pVmxTransient);
3998	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3999
4000	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4001	switch (rcVMRun)
4002	{
4003	case VERR_VMX_INVALID_VMXON_PTR:
4004	AssertFailed();
4005	break;
4006	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4007	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4008	{
4009	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4010	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4011	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4012	AssertRC(rc);
4013
4014	#ifdef VBOX_STRICT
4015	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4016	pVmxTransient->uExitReason));
4017	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4018	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4019	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4020	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4021	else
4022	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4023
4024	/* VMX control bits. */
4025	uint32_t u32Val;
4026	uint64_t u64Val;
4027	HMVMXHCUINTREG uHCReg;
4028	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4029	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4030	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4031	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4032	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4033	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4034	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4035	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4036	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4037	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4038	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4039	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4040	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4041	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4042	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4043	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4044	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4045	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4046	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4047	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4048	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4049	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4050	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4051	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4052	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4053	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4054	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4055	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4056	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4057	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4058	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4059	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4060	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4061	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4062	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4063	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4064	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4065	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4066	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4067	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4068	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4069	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4070
4071	/* Guest bits. */
4072	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4073	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4074	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4075	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4076	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4077	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4078	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4079	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4080
4081	/* Host bits. */
4082	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4083	Log4(("Host CR0 %#RHr\n", uHCReg));
4084	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4085	Log4(("Host CR3 %#RHr\n", uHCReg));
4086	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4087	Log4(("Host CR4 %#RHr\n", uHCReg));
4088
4089	RTGDTR HostGdtr;
4090	PCX86DESCHC pDesc;
4091	ASMGetGDTR(&HostGdtr);
4092	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4093	Log4(("Host CS %#08x\n", u32Val));
4094	if (u32Val < HostGdtr.cbGdt)
4095	{
4096	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4097	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4098	}
4099
4100	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4101	Log4(("Host DS %#08x\n", u32Val));
4102	if (u32Val < HostGdtr.cbGdt)
4103	{
4104	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4105	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4106	}
4107
4108	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4109	Log4(("Host ES %#08x\n", u32Val));
4110	if (u32Val < HostGdtr.cbGdt)
4111	{
4112	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4113	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4114	}
4115
4116	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4117	Log4(("Host FS %#08x\n", u32Val));
4118	if (u32Val < HostGdtr.cbGdt)
4119	{
4120	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4121	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4122	}
4123
4124	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4125	Log4(("Host GS %#08x\n", u32Val));
4126	if (u32Val < HostGdtr.cbGdt)
4127	{
4128	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4129	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4130	}
4131
4132	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4133	Log4(("Host SS %#08x\n", u32Val));
4134	if (u32Val < HostGdtr.cbGdt)
4135	{
4136	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4137	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4138	}
4139
4140	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4141	Log4(("Host TR %#08x\n", u32Val));
4142	if (u32Val < HostGdtr.cbGdt)
4143	{
4144	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4145	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4146	}
4147
4148	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4149	Log4(("Host TR Base %#RHv\n", uHCReg));
4150	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4151	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4152	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4153	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4154	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4155	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4156	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4157	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4159	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4160	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4161	Log4(("Host RSP %#RHv\n", uHCReg));
4162	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4163	Log4(("Host RIP %#RHv\n", uHCReg));
4164	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4165	if (HMVMX_IS_64BIT_HOST_MODE())
4166	{
4167	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4168	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4169	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4170	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4171	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4172	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4173	}
4174	# endif
4175	#endif /* VBOX_STRICT */
4176	break;
4177	}
4178
4179	default:
4180	/* Impossible */
4181	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4182	break;
4183	}
4184	NOREF(pVM);
4185	}
4186
4187
4188	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4189	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4190	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4191	#endif
4192	#ifdef VBOX_STRICT
4193	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4194	{
4195	switch (idxField)
4196	{
4197	case VMX_VMCS_GUEST_RIP:
4198	case VMX_VMCS_GUEST_RSP:
4199	case VMX_VMCS_GUEST_SYSENTER_EIP:
4200	case VMX_VMCS_GUEST_SYSENTER_ESP:
4201	case VMX_VMCS_GUEST_GDTR_BASE:
4202	case VMX_VMCS_GUEST_IDTR_BASE:
4203	case VMX_VMCS_GUEST_CS_BASE:
4204	case VMX_VMCS_GUEST_DS_BASE:
4205	case VMX_VMCS_GUEST_ES_BASE:
4206	case VMX_VMCS_GUEST_FS_BASE:
4207	case VMX_VMCS_GUEST_GS_BASE:
4208	case VMX_VMCS_GUEST_SS_BASE:
4209	case VMX_VMCS_GUEST_LDTR_BASE:
4210	case VMX_VMCS_GUEST_TR_BASE:
4211	case VMX_VMCS_GUEST_CR3:
4212	return true;
4213	}
4214	return false;
4215	}
4216
4217	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4218	{
4219	switch (idxField)
4220	{
4221	/* Read-only fields. */
4222	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4223	return true;
4224	}
4225	/* Remaining readable fields should also be writable. */
4226	return hmR0VmxIsValidWriteField(idxField);
4227	}
4228	#endif /* VBOX_STRICT */
4229
4230
4231	/**
4232	* Executes the specified handler in 64-bit mode.
4233	*
4234	* @returns VBox status code.
4235	* @param pVM Pointer to the VM.
4236	* @param pVCpu Pointer to the VMCPU.
4237	* @param pCtx Pointer to the guest CPU context.
4238	* @param enmOp The operation to perform.
4239	* @param cbParam Number of parameters.
4240	* @param paParam Array of 32-bit parameters.
4241	*/
4242	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4243	uint32_t *paParam)
4244	{
4245	int rc, rc2;
4246	PHMGLOBLCPUINFO pCpu;
4247	RTHCPHYS HCPhysCpuPage;
4248	RTCCUINTREG uOldEFlags;
4249
4250	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4251	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4252	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4253	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4254
4255	#ifdef VBOX_STRICT
4256	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4257	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4258
4259	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4260	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4261	#endif
4262
4263	/* Disable interrupts. */
4264	uOldEFlags = ASMIntDisableFlags();
4265
4266	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4267	RTCPUID idHostCpu = RTMpCpuId();
4268	CPUMR0SetLApic(pVM, idHostCpu);
4269	#endif
4270
4271	pCpu = HMR0GetCurrentCpu();
4272	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4273
4274	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4275	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4276
4277	/* Leave VMX Root Mode. */
4278	VMXDisable();
4279
4280	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4281
4282	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4283	CPUMSetHyperEIP(pVCpu, enmOp);
4284	for (int i = (int)cbParam - 1; i >= 0; i--)
4285	CPUMPushHyper(pVCpu, paParam[i]);
4286
4287	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4288
4289	/* Call the switcher. */
4290	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4291	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4292
4293	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4294	/* Make sure the VMX instructions don't cause #UD faults. */
4295	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4296
4297	/* Re-enter VMX Root Mode */
4298	rc2 = VMXEnable(HCPhysCpuPage);
4299	if (RT_FAILURE(rc2))
4300	{
4301	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4302	ASMSetFlags(uOldEFlags);
4303	return rc2;
4304	}
4305
4306	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4307	AssertRC(rc2);
4308	Assert(!(ASMGetFlags() & X86_EFL_IF));
4309	ASMSetFlags(uOldEFlags);
4310	return rc;
4311	}
4312
4313
4314	/**
4315	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4316	* supporting 64-bit guests.
4317	*
4318	* @returns VBox status code.
4319	* @param fResume Whether to VMLAUNCH or VMRESUME.
4320	* @param pCtx Pointer to the guest-CPU context.
4321	* @param pCache Pointer to the VMCS cache.
4322	* @param pVM Pointer to the VM.
4323	* @param pVCpu Pointer to the VMCPU.
4324	*/
4325	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4326	{
4327	uint32_t aParam[6];
4328	PHMGLOBLCPUINFO pCpu = NULL;
4329	RTHCPHYS HCPhysCpuPage = 0;
4330	int rc = VERR_INTERNAL_ERROR_5;
4331
4332	pCpu = HMR0GetCurrentCpu();
4333	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4334
4335	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4336	pCache->uPos = 1;
4337	pCache->interPD = PGMGetInterPaeCR3(pVM);
4338	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4339	#endif
4340
4341	#ifdef VBOX_STRICT
4342	pCache->TestIn.HCPhysCpuPage = 0;
4343	pCache->TestIn.HCPhysVmcs = 0;
4344	pCache->TestIn.pCache = 0;
4345	pCache->TestOut.HCPhysVmcs = 0;
4346	pCache->TestOut.pCache = 0;
4347	pCache->TestOut.pCtx = 0;
4348	pCache->TestOut.eflags = 0;
4349	#endif
4350
4351	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4352	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4353	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4354	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4355	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4356	aParam[5] = 0;
4357
4358	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4359	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4360	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4361	#endif
4362	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4363
4364	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4365	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4366	Assert(pCtx->dr[4] == 10);
4367	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4368	#endif
4369
4370	#ifdef VBOX_STRICT
4371	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4372	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4373	pVCpu->hm.s.vmx.HCPhysVmcs));
4374	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4375	pCache->TestOut.HCPhysVmcs));
4376	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4377	pCache->TestOut.pCache));
4378	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4379	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4380	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4381	pCache->TestOut.pCtx));
4382	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4383	#endif
4384	return rc;
4385	}
4386
4387
4388	/**
4389	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4390	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4391	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4392	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4393	*
4394	* @returns VBox status code.
4395	* @param pVM Pointer to the VM.
4396	* @param pVCpu Pointer to the VMCPU.
4397	*/
4398	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4399	{
4400	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4401	{ \
4402	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4403	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4404	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4405	++cReadFields; \
4406	}
4407
4408	AssertPtr(pVM);
4409	AssertPtr(pVCpu);
4410	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4411	uint32_t cReadFields = 0;
4412
4413	/*
4414	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4415	* and serve to indicate exceptions to the rules.
4416	*/
4417
4418	/* Guest-natural selector base fields. */
4419	#if 0
4420	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4421	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4422	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4423	#endif
4424	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4425	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4426	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4427	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4428	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4429	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4430	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4431	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4432	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4433	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4434	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4435	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4436	#if 0
4437	/* Unused natural width guest-state fields. */
4438	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4439	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4440	#endif
4441	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4442	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4443
4444	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4445	#if 0
4446	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4447	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4448	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4449	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4450	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4451	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4452	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4453	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4454	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4455	#endif
4456
4457	/* Natural width guest-state fields. */
4458	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4459	#if 0
4460	/* Currently unused field. */
4461	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4462	#endif
4463
4464	if (pVM->hm.s.fNestedPaging)
4465	{
4466	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4467	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4468	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4469	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4470	}
4471	else
4472	{
4473	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4474	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4475	}
4476
4477	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4478	return VINF_SUCCESS;
4479	}
4480
4481
4482	/**
4483	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4484	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4485	* darwin, running 64-bit guests).
4486	*
4487	* @returns VBox status code.
4488	* @param pVCpu Pointer to the VMCPU.
4489	* @param idxField The VMCS field encoding.
4490	* @param u64Val 16, 32 or 64 bits value.
4491	*/
4492	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4493	{
4494	int rc;
4495	switch (idxField)
4496	{
4497	/*
4498	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4499	*/
4500	/* 64-bit Control fields. */
4501	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4502	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4503	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4504	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4505	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4506	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4507	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4508	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4509	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4510	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4511	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4512	case VMX_VMCS64_CTRL_EPTP_FULL:
4513	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4514	/* 64-bit Guest-state fields. */
4515	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4516	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4517	case VMX_VMCS64_GUEST_PAT_FULL:
4518	case VMX_VMCS64_GUEST_EFER_FULL:
4519	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4520	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4521	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4522	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4523	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4524	/* 64-bit Host-state fields. */
4525	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4526	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4527	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4528	{
4529	rc = VMXWriteVmcs32(idxField, u64Val);
4530	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4531	break;
4532	}
4533
4534	/*
4535	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4536	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4537	*/
4538	/* Natural-width Guest-state fields. */
4539	case VMX_VMCS_GUEST_CR3:
4540	case VMX_VMCS_GUEST_ES_BASE:
4541	case VMX_VMCS_GUEST_CS_BASE:
4542	case VMX_VMCS_GUEST_SS_BASE:
4543	case VMX_VMCS_GUEST_DS_BASE:
4544	case VMX_VMCS_GUEST_FS_BASE:
4545	case VMX_VMCS_GUEST_GS_BASE:
4546	case VMX_VMCS_GUEST_LDTR_BASE:
4547	case VMX_VMCS_GUEST_TR_BASE:
4548	case VMX_VMCS_GUEST_GDTR_BASE:
4549	case VMX_VMCS_GUEST_IDTR_BASE:
4550	case VMX_VMCS_GUEST_RSP:
4551	case VMX_VMCS_GUEST_RIP:
4552	case VMX_VMCS_GUEST_SYSENTER_ESP:
4553	case VMX_VMCS_GUEST_SYSENTER_EIP:
4554	{
4555	if (!(u64Val >> 32))
4556	{
4557	/* If this field is 64-bit, VT-x will zero out the top bits. */
4558	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4559	}
4560	else
4561	{
4562	/* Assert that only the 32->64 switcher case should ever come here. */
4563	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4564	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4565	}
4566	break;
4567	}
4568
4569	default:
4570	{
4571	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4572	rc = VERR_INVALID_PARAMETER;
4573	break;
4574	}
4575	}
4576	AssertRCReturn(rc, rc);
4577	return rc;
4578	}
4579
4580
4581	/**
4582	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4583	* hosts (except darwin) for 64-bit guests.
4584	*
4585	* @param pVCpu Pointer to the VMCPU.
4586	* @param idxField The VMCS field encoding.
4587	* @param u64Val 16, 32 or 64 bits value.
4588	*/
4589	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4590	{
4591	AssertPtr(pVCpu);
4592	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4593
4594	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4595	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4596
4597	/* Make sure there are no duplicates. */
4598	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4599	{
4600	if (pCache->Write.aField[i] == idxField)
4601	{
4602	pCache->Write.aFieldVal[i] = u64Val;
4603	return VINF_SUCCESS;
4604	}
4605	}
4606
4607	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4608	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4609	pCache->Write.cValidEntries++;
4610	return VINF_SUCCESS;
4611	}
4612
4613	/* Enable later when the assembly code uses these as callbacks. */
4614	#if 0
4615	/*
4616	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4617	*
4618	* @param pVCpu Pointer to the VMCPU.
4619	* @param pCache Pointer to the VMCS cache.
4620	*
4621	* @remarks No-long-jump zone!!!
4622	*/
4623	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4624	{
4625	AssertPtr(pCache);
4626	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4627	{
4628	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4629	AssertRC(rc);
4630	}
4631	pCache->Write.cValidEntries = 0;
4632	}
4633
4634
4635	/**
4636	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4637	*
4638	* @param pVCpu Pointer to the VMCPU.
4639	* @param pCache Pointer to the VMCS cache.
4640	*
4641	* @remarks No-long-jump zone!!!
4642	*/
4643	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4644	{
4645	AssertPtr(pCache);
4646	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4647	{
4648	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4649	AssertRC(rc);
4650	}
4651	}
4652	#endif
4653	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4654
4655
4656	/**
4657	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4658	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4659	* timer.
4660	*
4661	* @returns VBox status code.
4662	* @param pVCpu Pointer to the VMCPU.
4663	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4664	* out-of-sync. Make sure to update the required fields
4665	* before using them.
4666	* @remarks No-long-jump zone!!!
4667	*/
4668	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4669	{
4670	int rc = VERR_INTERNAL_ERROR_5;
4671	bool fOffsettedTsc = false;
4672	PVM pVM = pVCpu->CTX_SUFF(pVM);
4673	if (pVM->hm.s.vmx.fUsePreemptTimer)
4674	{
4675	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4676
4677	/* Make sure the returned values have sane upper and lower boundaries. */
4678	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4679	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4680	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4681	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4682
4683	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4684	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4685	}
4686	else
4687	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4688
4689	if (fOffsettedTsc)
4690	{
4691	uint64_t u64CurTSC = ASMReadTSC();
4692	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4693	{
4694	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4695	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4696
4697	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4698	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4699	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4700	}
4701	else
4702	{
4703	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4704	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4706	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4707	}
4708	}
4709	else
4710	{
4711	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4712	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4713	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4714	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4715	}
4716	}
4717
4718
4719	/**
4720	* Determines if an exception is a contributory exception. Contributory
4721	* exceptions are ones which can cause double-faults. Page-fault is
4722	* intentionally not included here as it's a conditional contributory exception.
4723	*
4724	* @returns true if the exception is contributory, false otherwise.
4725	* @param uVector The exception vector.
4726	*/
4727	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4728	{
4729	switch (uVector)
4730	{
4731	case X86_XCPT_GP:
4732	case X86_XCPT_SS:
4733	case X86_XCPT_NP:
4734	case X86_XCPT_TS:
4735	case X86_XCPT_DE:
4736	return true;
4737	default:
4738	break;
4739	}
4740	return false;
4741	}
4742
4743
4744	/**
4745	* Sets an event as a pending event to be injected into the guest.
4746	*
4747	* @param pVCpu Pointer to the VMCPU.
4748	* @param u32IntrInfo The VM-entry interruption-information field.
4749	* @param cbInstr The VM-entry instruction length in bytes (for software
4750	* interrupts, exceptions and privileged software
4751	* exceptions).
4752	* @param u32ErrCode The VM-entry exception error code.
4753	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4754	* page-fault.
4755	*
4756	* @remarks Statistics counter assumes this is a guest event being injected or
4757	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4758	* always incremented.
4759	*/
4760	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4761	RTGCUINTPTR GCPtrFaultAddress)
4762	{
4763	Assert(!pVCpu->hm.s.Event.fPending);
4764	pVCpu->hm.s.Event.fPending = true;
4765	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4766	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4767	pVCpu->hm.s.Event.cbInstr = cbInstr;
4768	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4769
4770	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4771	}
4772
4773
4774	/**
4775	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4776	*
4777	* @param pVCpu Pointer to the VMCPU.
4778	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4779	* out-of-sync. Make sure to update the required fields
4780	* before using them.
4781	*/
4782	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4783	{
4784	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4785	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4786	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4787	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4788	}
4789
4790
4791	/**
4792	* Handle a condition that occurred while delivering an event through the guest
4793	* IDT.
4794	*
4795	* @returns VBox status code (informational error codes included).
4796	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4797	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4798	* continue execution of the guest which will delivery the #DF.
4799	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4800	*
4801	* @param pVCpu Pointer to the VMCPU.
4802	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4803	* out-of-sync. Make sure to update the required fields
4804	* before using them.
4805	* @param pVmxTransient Pointer to the VMX transient structure.
4806	*
4807	* @remarks No-long-jump zone!!!
4808	*/
4809	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4810	{
4811	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4812	AssertRC(rc);
4813	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4814	{
4815	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4816	AssertRCReturn(rc, rc);
4817
4818	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4819	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4820	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4821
4822	typedef enum
4823	{
4824	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4825	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4826	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4827	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4828	} VMXREFLECTXCPT;
4829
4830	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4831	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4832	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4833	{
4834	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4835	{
4836	enmReflect = VMXREFLECTXCPT_XCPT;
4837	#ifdef VBOX_STRICT
4838	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4839	&& uExitVector == X86_XCPT_PF)
4840	{
4841	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4842	}
4843	#endif
4844	if ( uExitVector == X86_XCPT_PF
4845	&& uIdtVector == X86_XCPT_PF)
4846	{
4847	pVmxTransient->fVectoringPF = true;
4848	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4849	}
4850	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4851	&& hmR0VmxIsContributoryXcpt(uExitVector)
4852	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4853	\|\| uIdtVector == X86_XCPT_PF))
4854	{
4855	enmReflect = VMXREFLECTXCPT_DF;
4856	}
4857	else if (uIdtVector == X86_XCPT_DF)
4858	enmReflect = VMXREFLECTXCPT_TF;
4859	}
4860	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4861	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4862	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4863	{
4864	/*
4865	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4866	* (whatever they are) as they reoccur when restarting the instruction.
4867	*/
4868	enmReflect = VMXREFLECTXCPT_XCPT;
4869	}
4870	}
4871	else
4872	{
4873	/*
4874	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4875	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4876	* original exception to the guest after handling the VM-exit.
4877	*/
4878	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4879	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4880	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4881	{
4882	enmReflect = VMXREFLECTXCPT_XCPT;
4883	}
4884	}
4885
4886	switch (enmReflect)
4887	{
4888	case VMXREFLECTXCPT_XCPT:
4889	{
4890	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4891	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4892	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
4893
4894	uint32_t u32ErrCode = 0;
4895	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
4896	{
4897	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4898	AssertRCReturn(rc, rc);
4899	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4900	}
4901
4902	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4903	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4904	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4905	rc = VINF_SUCCESS;
4906	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
4907	pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4908
4909	break;
4910	}
4911
4912	case VMXREFLECTXCPT_DF:
4913	{
4914	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4915	rc = VINF_HM_DOUBLE_FAULT;
4916	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
4917	pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4918
4919	break;
4920	}
4921
4922	case VMXREFLECTXCPT_TF:
4923	{
4924	rc = VINF_EM_RESET;
4925	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
4926	uExitVector));
4927	break;
4928	}
4929
4930	default:
4931	Assert(rc == VINF_SUCCESS);
4932	break;
4933	}
4934	}
4935	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4942	*
4943	* @returns VBox status code.
4944	* @param pVCpu Pointer to the VMCPU.
4945	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4946	* out-of-sync. Make sure to update the required fields
4947	* before using them.
4948	*
4949	* @remarks No-long-jump zone!!!
4950	*/
4951	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4952	{
4953	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4954	{
4955	uint32_t uVal = 0;
4956	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4957	AssertRCReturn(rc, rc);
4958	uint32_t uShadow = 0;
4959	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4960	AssertRCReturn(rc, rc);
4961
4962	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4963	CPUMSetGuestCR0(pVCpu, uVal);
4964	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4965	}
4966	return VINF_SUCCESS;
4967	}
4968
4969
4970	/**
4971	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4972	*
4973	* @returns VBox status code.
4974	* @param pVCpu Pointer to the VMCPU.
4975	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4976	* out-of-sync. Make sure to update the required fields
4977	* before using them.
4978	*
4979	* @remarks No-long-jump zone!!!
4980	*/
4981	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4982	{
4983	int rc = VINF_SUCCESS;
4984	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4985	{
4986	uint32_t uVal = 0;
4987	uint32_t uShadow = 0;
4988	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4989	AssertRCReturn(rc, rc);
4990	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4991	AssertRCReturn(rc, rc);
4992
4993	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4994	CPUMSetGuestCR4(pVCpu, uVal);
4995	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4996	}
4997	return rc;
4998	}
4999
5000
5001	/**
5002	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5003	*
5004	* @returns VBox status code.
5005	* @param pVCpu Pointer to the VMCPU.
5006	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5007	* out-of-sync. Make sure to update the required fields
5008	* before using them.
5009	*
5010	* @remarks No-long-jump zone!!!
5011	*/
5012	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5013	{
5014	int rc = VINF_SUCCESS;
5015	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5016	{
5017	uint64_t u64Val = 0;
5018	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5019	AssertRCReturn(rc, rc);
5020
5021	pMixedCtx->rip = u64Val;
5022	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5023	}
5024	return rc;
5025	}
5026
5027
5028	/**
5029	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5030	*
5031	* @returns VBox status code.
5032	* @param pVCpu Pointer to the VMCPU.
5033	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5034	* out-of-sync. Make sure to update the required fields
5035	* before using them.
5036	*
5037	* @remarks No-long-jump zone!!!
5038	*/
5039	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5040	{
5041	int rc = VINF_SUCCESS;
5042	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5043	{
5044	uint64_t u64Val = 0;
5045	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5046	AssertRCReturn(rc, rc);
5047
5048	pMixedCtx->rsp = u64Val;
5049	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5050	}
5051	return rc;
5052	}
5053
5054
5055	/**
5056	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5057	*
5058	* @returns VBox status code.
5059	* @param pVCpu Pointer to the VMCPU.
5060	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5061	* out-of-sync. Make sure to update the required fields
5062	* before using them.
5063	*
5064	* @remarks No-long-jump zone!!!
5065	*/
5066	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5067	{
5068	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5069	{
5070	uint32_t uVal = 0;
5071	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5072	AssertRCReturn(rc, rc);
5073
5074	pMixedCtx->eflags.u32 = uVal;
5075	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5076	{
5077	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5078	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5079
5080	pMixedCtx->eflags.Bits.u1VM = 0;
5081	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5082	}
5083
5084	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5085	}
5086	return VINF_SUCCESS;
5087	}
5088
5089
5090	/**
5091	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5092	* guest-CPU context.
5093	*/
5094	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5095	{
5096	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5097	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5098	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5099	return rc;
5100	}
5101
5102
5103	/**
5104	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5105	* from the guest-state area in the VMCS.
5106	*
5107	* @param pVCpu Pointer to the VMCPU.
5108	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5109	* out-of-sync. Make sure to update the required fields
5110	* before using them.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	*/
5114	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5115	{
5116	uint32_t uIntrState = 0;
5117	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5118	AssertRC(rc);
5119
5120	if (!uIntrState)
5121	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5122	else
5123	{
5124	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5125	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5126	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5127	AssertRC(rc);
5128	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5129	AssertRC(rc);
5130
5131	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5132	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5133	}
5134	}
5135
5136
5137	/**
5138	* Saves the guest's activity state.
5139	*
5140	* @returns VBox status code.
5141	* @param pVCpu Pointer to the VMCPU.
5142	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5143	* out-of-sync. Make sure to update the required fields
5144	* before using them.
5145	*
5146	* @remarks No-long-jump zone!!!
5147	*/
5148	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5149	{
5150	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5151	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5152	return VINF_SUCCESS;
5153	}
5154
5155
5156	/**
5157	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5158	* the current VMCS into the guest-CPU context.
5159	*
5160	* @returns VBox status code.
5161	* @param pVCpu Pointer to the VMCPU.
5162	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5163	* out-of-sync. Make sure to update the required fields
5164	* before using them.
5165	*
5166	* @remarks No-long-jump zone!!!
5167	*/
5168	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5169	{
5170	int rc = VINF_SUCCESS;
5171	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5172	{
5173	uint32_t u32Val = 0;
5174	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5175	pMixedCtx->SysEnter.cs = u32Val;
5176	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5177	}
5178
5179	uint64_t u64Val = 0;
5180	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5181	{
5182	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5183	pMixedCtx->SysEnter.eip = u64Val;
5184	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5185	}
5186	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5187	{
5188	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5189	pMixedCtx->SysEnter.esp = u64Val;
5190	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5191	}
5192	return rc;
5193	}
5194
5195
5196	/**
5197	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5198	* context.
5199	*
5200	* @returns VBox status code.
5201	* @param pVCpu Pointer to the VMCPU.
5202	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5203	* out-of-sync. Make sure to update the required fields
5204	* before using them.
5205	*
5206	* @remarks No-long-jump zone!!!
5207	*/
5208	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5209	{
5210	int rc = VINF_SUCCESS;
5211	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5212	{
5213	uint64_t u64Val = 0;
5214	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5215	pMixedCtx->fs.u64Base = u64Val;
5216	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5217	}
5218	return rc;
5219	}
5220
5221
5222	/**
5223	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5224	* context.
5225	*
5226	* @returns VBox status code.
5227	* @param pVCpu Pointer to the VMCPU.
5228	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5229	* out-of-sync. Make sure to update the required fields
5230	* before using them.
5231	*
5232	* @remarks No-long-jump zone!!!
5233	*/
5234	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5235	{
5236	int rc = VINF_SUCCESS;
5237	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5238	{
5239	uint64_t u64Val = 0;
5240	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5241	pMixedCtx->gs.u64Base = u64Val;
5242	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5243	}
5244	return rc;
5245	}
5246
5247
5248	/**
5249	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5250	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5251	* and TSC_AUX.
5252	*
5253	* @returns VBox status code.
5254	* @param pVCpu Pointer to the VMCPU.
5255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5256	* out-of-sync. Make sure to update the required fields
5257	* before using them.
5258	*
5259	* @remarks No-long-jump zone!!!
5260	*/
5261	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5262	{
5263	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5264	return VINF_SUCCESS;
5265
5266	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5267	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5268	{
5269	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5270	pMsr += i;
5271	switch (pMsr->u32IndexMSR)
5272	{
5273	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5274	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5275	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5276	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5277	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5278	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5279	default:
5280	{
5281	AssertFailed();
5282	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5283	}
5284	}
5285	}
5286	#endif
5287
5288	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5289	return VINF_SUCCESS;
5290	}
5291
5292
5293	/**
5294	* Saves the guest control registers from the current VMCS into the guest-CPU
5295	* context.
5296	*
5297	* @returns VBox status code.
5298	* @param pVCpu Pointer to the VMCPU.
5299	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5300	* out-of-sync. Make sure to update the required fields
5301	* before using them.
5302	*
5303	* @remarks No-long-jump zone!!!
5304	*/
5305	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5306	{
5307	/* Guest CR0. Guest FPU. */
5308	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5309	AssertRCReturn(rc, rc);
5310
5311	/* Guest CR4. */
5312	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5313	AssertRCReturn(rc, rc);
5314
5315	/* Guest CR2 - updated always during the world-switch or in #PF. */
5316	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5317	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5318	{
5319	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5320	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5321
5322	PVM pVM = pVCpu->CTX_SUFF(pVM);
5323	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5324	\|\| ( pVM->hm.s.fNestedPaging
5325	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5326	{
5327	uint64_t u64Val = 0;
5328	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5329	if (pMixedCtx->cr3 != u64Val)
5330	{
5331	CPUMSetGuestCR3(pVCpu, u64Val);
5332	if (VMMRZCallRing3IsEnabled(pVCpu))
5333	{
5334	PGMUpdateCR3(pVCpu, u64Val);
5335	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5336	}
5337	else
5338	{
5339	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5340	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5341	}
5342	}
5343
5344	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5345	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5346	{
5347	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5348	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5349	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5350	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5351
5352	if (VMMRZCallRing3IsEnabled(pVCpu))
5353	{
5354	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5355	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5356	}
5357	else
5358	{
5359	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5360	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5361	}
5362	}
5363	}
5364
5365	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5366	}
5367
5368	/*
5369	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5370	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5371	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5372	*
5373	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5374	*/
5375	if (VMMRZCallRing3IsEnabled(pVCpu))
5376	{
5377	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5378	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5379
5380	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5381	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5382
5383	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5384	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5385	}
5386
5387	return rc;
5388	}
5389
5390
5391	/**
5392	* Reads a guest segment register from the current VMCS into the guest-CPU
5393	* context.
5394	*
5395	* @returns VBox status code.
5396	* @param pVCpu Pointer to the VMCPU.
5397	* @param idxSel Index of the selector in the VMCS.
5398	* @param idxLimit Index of the segment limit in the VMCS.
5399	* @param idxBase Index of the segment base in the VMCS.
5400	* @param idxAccess Index of the access rights of the segment in the VMCS.
5401	* @param pSelReg Pointer to the segment selector.
5402	*
5403	* @remarks No-long-jump zone!!!
5404	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5405	* macro as that takes care of whether to read from the VMCS cache or
5406	* not.
5407	*/
5408	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5409	PCPUMSELREG pSelReg)
5410	{
5411	uint32_t u32Val = 0;
5412	int rc = VMXReadVmcs32(idxSel, &u32Val);
5413	AssertRCReturn(rc, rc);
5414	pSelReg->Sel = (uint16_t)u32Val;
5415	pSelReg->ValidSel = (uint16_t)u32Val;
5416	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5417
5418	rc = VMXReadVmcs32(idxLimit, &u32Val);
5419	AssertRCReturn(rc, rc);
5420	pSelReg->u32Limit = u32Val;
5421
5422	uint64_t u64Val = 0;
5423	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5424	AssertRCReturn(rc, rc);
5425	pSelReg->u64Base = u64Val;
5426
5427	rc = VMXReadVmcs32(idxAccess, &u32Val);
5428	AssertRCReturn(rc, rc);
5429	pSelReg->Attr.u = u32Val;
5430
5431	/*
5432	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5433	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5434	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5435	*
5436	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5437	*/
5438	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5439	{
5440	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5441	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5442	}
5443	return VINF_SUCCESS;
5444	}
5445
5446
5447	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5448	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5449	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5450	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5451	#else
5452	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5453	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5454	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5455	#endif
5456
5457
5458	/**
5459	* Saves the guest segment registers from the current VMCS into the guest-CPU
5460	* context.
5461	*
5462	* @returns VBox status code.
5463	* @param pVCpu Pointer to the VMCPU.
5464	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5465	* out-of-sync. Make sure to update the required fields
5466	* before using them.
5467	*
5468	* @remarks No-long-jump zone!!!
5469	*/
5470	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5471	{
5472	/* Guest segment registers. */
5473	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5474	{
5475	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5476	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5477	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5478	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5479	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5480	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5481	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5482
5483	/* Restore segment attributes for real-on-v86 mode hack. */
5484	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5485	{
5486	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5487	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5488	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5489	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5490	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5491	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5492	}
5493	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5494	}
5495
5496	return VINF_SUCCESS;
5497	}
5498
5499
5500	/**
5501	* Saves the guest descriptor table registers and task register from the current
5502	* VMCS into the guest-CPU context.
5503	*
5504	* @returns VBox status code.
5505	* @param pVCpu Pointer to the VMCPU.
5506	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5507	* out-of-sync. Make sure to update the required fields
5508	* before using them.
5509	*
5510	* @remarks No-long-jump zone!!!
5511	*/
5512	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5513	{
5514	int rc = VINF_SUCCESS;
5515
5516	/* Guest LDTR. */
5517	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5518	{
5519	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5520	AssertRCReturn(rc, rc);
5521	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5522	}
5523
5524	/* Guest GDTR. */
5525	uint64_t u64Val = 0;
5526	uint32_t u32Val = 0;
5527	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5528	{
5529	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5530	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5531	pMixedCtx->gdtr.pGdt = u64Val;
5532	pMixedCtx->gdtr.cbGdt = u32Val;
5533	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5534	}
5535
5536	/* Guest IDTR. */
5537	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5538	{
5539	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5540	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5541	pMixedCtx->idtr.pIdt = u64Val;
5542	pMixedCtx->idtr.cbIdt = u32Val;
5543	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5544	}
5545
5546	/* Guest TR. */
5547	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5548	{
5549	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5550	AssertRCReturn(rc, rc);
5551
5552	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5553	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5554	{
5555	rc = VMXLOCAL_READ_SEG(TR, tr);
5556	AssertRCReturn(rc, rc);
5557	}
5558	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5559	}
5560	return rc;
5561	}
5562
5563	#undef VMXLOCAL_READ_SEG
5564
5565
5566	/**
5567	* Saves the guest debug registers from the current VMCS into the guest-CPU
5568	* context.
5569	*
5570	* @returns VBox status code.
5571	* @param pVCpu Pointer to the VMCPU.
5572	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5573	* out-of-sync. Make sure to update the required fields
5574	* before using them.
5575	*
5576	* @remarks No-long-jump zone!!!
5577	*/
5578	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5579	{
5580	int rc = VINF_SUCCESS;
5581	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5582	{
5583	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5584	uint32_t u32Val;
5585	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5586	pMixedCtx->dr[7] = u32Val;
5587
5588	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5589	}
5590	return rc;
5591	}
5592
5593
5594	/**
5595	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5596	*
5597	* @returns VBox status code.
5598	* @param pVCpu Pointer to the VMCPU.
5599	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5600	* out-of-sync. Make sure to update the required fields
5601	* before using them.
5602	*
5603	* @remarks No-long-jump zone!!!
5604	*/
5605	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5606	{
5607	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5608	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5609	return VINF_SUCCESS;
5610	}
5611
5612
5613	/**
5614	* Saves the entire guest state from the currently active VMCS into the
5615	* guest-CPU context. This essentially VMREADs all guest-data.
5616	*
5617	* @returns VBox status code.
5618	* @param pVCpu Pointer to the VMCPU.
5619	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5620	* out-of-sync. Make sure to update the required fields
5621	* before using them.
5622	*/
5623	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5624	{
5625	Assert(pVCpu);
5626	Assert(pMixedCtx);
5627
5628	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5629	return VINF_SUCCESS;
5630
5631	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5632	there is no real need to. */
5633	if (VMMRZCallRing3IsEnabled(pVCpu))
5634	VMMR0LogFlushDisable(pVCpu);
5635	else
5636	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5637	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5638
5639	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5640	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5641
5642	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5643	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5644
5645	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5646	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5647
5648	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5649	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5650
5651	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5652	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5653
5654	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5655	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5656
5657	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5658	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5659
5660	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5662
5663	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5664	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5665
5666	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5667	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5668
5669	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5670	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5671
5672	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5673	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5674
5675	if (VMMRZCallRing3IsEnabled(pVCpu))
5676	VMMR0LogFlushEnable(pVCpu);
5677
5678	return rc;
5679	}
5680
5681
5682	/**
5683	* Check per-VM and per-VCPU force flag actions that require us to go back to
5684	* ring-3 for one reason or another.
5685	*
5686	* @returns VBox status code (information status code included).
5687	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5688	* ring-3.
5689	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5690	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5691	* interrupts)
5692	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5693	* all EMTs to be in ring-3.
5694	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5695	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5696	* to the EM loop.
5697	*
5698	* @param pVM Pointer to the VM.
5699	* @param pVCpu Pointer to the VMCPU.
5700	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5701	* out-of-sync. Make sure to update the required fields
5702	* before using them.
5703	*/
5704	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5707
5708	int rc = VERR_INTERNAL_ERROR_5;
5709	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5710	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5711	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5712	{
5713	/* We need the control registers now, make sure the guest-CPU context is updated. */
5714	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5715	AssertRCReturn(rc, rc);
5716
5717	/* Pending HM CR3 sync. */
5718	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5719	{
5720	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5721	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5722	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5723	}
5724
5725	/* Pending HM PAE PDPEs. */
5726	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5727	{
5728	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5729	AssertRC(rc);
5730	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5731	}
5732
5733	/* Pending PGM C3 sync. */
5734	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5735	{
5736	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5737	if (rc != VINF_SUCCESS)
5738	{
5739	AssertRC(rc);
5740	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5741	return rc;
5742	}
5743	}
5744
5745	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5746	/* -XXX- what was that about single stepping? */
5747	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5748	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5749	{
5750	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5751	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5752	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5753	return rc;
5754	}
5755
5756	/* Pending VM request packets, such as hardware interrupts. */
5757	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5758	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5759	{
5760	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5761	return VINF_EM_PENDING_REQUEST;
5762	}
5763
5764	/* Pending PGM pool flushes. */
5765	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5766	{
5767	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5768	return VINF_PGM_POOL_FLUSH_PENDING;
5769	}
5770
5771	/* Pending DMA requests. */
5772	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5773	{
5774	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5775	return VINF_EM_RAW_TO_R3;
5776	}
5777	}
5778
5779	/* Paranoia. */
5780	Assert(rc != VERR_EM_INTERPRETER);
5781	return VINF_SUCCESS;
5782	}
5783
5784
5785	/**
5786	* Converts any TRPM trap into a pending HM event. This is typically used when
5787	* entering from ring-3 (not longjmp returns).
5788	*
5789	* @param pVCpu Pointer to the VMCPU.
5790	*/
5791	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5792	{
5793	Assert(TRPMHasTrap(pVCpu));
5794	Assert(!pVCpu->hm.s.Event.fPending);
5795
5796	uint8_t uVector;
5797	TRPMEVENT enmTrpmEvent;
5798	RTGCUINT uErrCode;
5799	RTGCUINTPTR GCPtrFaultAddress;
5800	uint8_t cbInstr;
5801
5802	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5803	AssertRC(rc);
5804
5805	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5806	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5807	if (enmTrpmEvent == TRPM_TRAP)
5808	{
5809	switch (uVector)
5810	{
5811	case X86_XCPT_BP:
5812	case X86_XCPT_OF:
5813	{
5814	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5815	break;
5816	}
5817
5818	case X86_XCPT_PF:
5819	case X86_XCPT_DF:
5820	case X86_XCPT_TS:
5821	case X86_XCPT_NP:
5822	case X86_XCPT_SS:
5823	case X86_XCPT_GP:
5824	case X86_XCPT_AC:
5825	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5826	/* no break! */
5827	default:
5828	{
5829	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5830	break;
5831	}
5832	}
5833	}
5834	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5835	{
5836	if (uVector == X86_XCPT_NMI)
5837	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5838	else
5839	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5840	}
5841	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5842	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5843	else
5844	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5845
5846	rc = TRPMResetTrap(pVCpu);
5847	AssertRC(rc);
5848	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5849	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5850
5851	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5852	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5853	}
5854
5855
5856	/**
5857	* Converts any pending HM event into a TRPM trap. Typically used when leaving
5858	* VT-x to execute any instruction.
5859	*
5860	* @param pvCpu Pointer to the VMCPU.
5861	*/
5862	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5863	{
5864	Assert(pVCpu->hm.s.Event.fPending);
5865
5866	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5867	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5868	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5869	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5870
5871	/* If a trap was already pending, we did something wrong! */
5872	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5873
5874	TRPMEVENT enmTrapType;
5875	switch (uVectorType)
5876	{
5877	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5878	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5879	enmTrapType = TRPM_HARDWARE_INT;
5880	break;
5881	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5882	enmTrapType = TRPM_SOFTWARE_INT;
5883	break;
5884	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5885	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5886	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5887	enmTrapType = TRPM_TRAP;
5888	break;
5889	default:
5890	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5891	enmTrapType = TRPM_32BIT_HACK;
5892	break;
5893	}
5894
5895	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5896
5897	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5898	AssertRC(rc);
5899
5900	if (fErrorCodeValid)
5901	TRPMSetErrorCode(pVCpu, uErrorCode);
5902
5903	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5904	&& uVector == X86_XCPT_PF)
5905	{
5906	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5907	}
5908	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5909	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5910	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5911	{
5912	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5913	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5914	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5915	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5916	}
5917	pVCpu->hm.s.Event.fPending = false;
5918	}
5919
5920
5921	/**
5922	* Does the necessary state syncing before doing a longjmp to ring-3.
5923	*
5924	* @param pVM Pointer to the VM.
5925	* @param pVCpu Pointer to the VMCPU.
5926	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5927	* out-of-sync. Make sure to update the required fields
5928	* before using them.
5929	* @param rcExit The reason for exiting to ring-3. Can be
5930	* VINF_VMM_UNKNOWN_RING3_CALL.
5931	*
5932	* @remarks No-long-jmp zone!!!
5933	*/
5934	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5935	{
5936	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5937	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5938
5939	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5940	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5941	AssertRC(rc);
5942
5943	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
5944	if (CPUMIsGuestFPUStateActive(pVCpu))
5945	{
5946	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5947	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5948	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5949	}
5950
5951	/* Restore host debug registers if necessary and resync on next R0 reentry. */
5952	if (CPUMIsGuestDebugStateActive(pVCpu))
5953	{
5954	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5955	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5956	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5957	}
5958	else if (CPUMIsHyperDebugStateActive(pVCpu))
5959	{
5960	CPUMR0LoadHostDebugState(pVM, pVCpu);
5961	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5962	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5963	}
5964
5965	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
5966	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
5967	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
5968	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
5969	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
5970	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
5971	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
5972	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5973
5974	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5975	}
5976
5977
5978	/**
5979	* An action requires us to go back to ring-3. This function does the necessary
5980	* steps before we can safely return to ring-3. This is not the same as longjmps
5981	* to ring-3, this is voluntary.
5982	*
5983	* @param pVM Pointer to the VM.
5984	* @param pVCpu Pointer to the VMCPU.
5985	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5986	* out-of-sync. Make sure to update the required fields
5987	* before using them.
5988	* @param rcExit The reason for exiting to ring-3. Can be
5989	* VINF_VMM_UNKNOWN_RING3_CALL.
5990	*/
5991	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5992	{
5993	Assert(pVM);
5994	Assert(pVCpu);
5995	Assert(pMixedCtx);
5996	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5997
5998	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5999	{
6000	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6001	return;
6002	}
6003	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6004	{
6005	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6006	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6007	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6008	pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6009	return;
6010	}
6011
6012	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6013	VMMRZCallRing3Disable(pVCpu);
6014	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6015
6016	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6017	if (pVCpu->hm.s.Event.fPending)
6018	{
6019	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6020	Assert(!pVCpu->hm.s.Event.fPending);
6021	}
6022
6023	/* Sync. the guest state. */
6024	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
6025	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6026
6027	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6028	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6029	\| CPUM_CHANGED_LDTR
6030	\| CPUM_CHANGED_GDTR
6031	\| CPUM_CHANGED_IDTR
6032	\| CPUM_CHANGED_TR
6033	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6034
6035	/* On our way back from ring-3 the following needs to be done. */
6036	/** @todo This can change with preemption hooks. */
6037	if (rcExit == VINF_EM_RAW_INTERRUPT)
6038	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
6039	else
6040	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
6041
6042	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6043	VMMRZCallRing3Enable(pVCpu);
6044	}
6045
6046
6047	/**
6048	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6049	* longjump to ring-3 and possibly get preempted.
6050	*
6051	* @param pVCpu Pointer to the VMCPU.
6052	* @param enmOperation The operation causing the ring-3 longjump.
6053	* @param pvUser The user argument (pointer to the possibly
6054	* out-of-date guest-CPU context).
6055	*
6056	* @remarks Must never be called with @a enmOperation ==
6057	* VMMCALLRING3_VM_R0_ASSERTION.
6058	*/
6059	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6060	{
6061	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6062	Assert(pVCpu);
6063	Assert(pvUser);
6064	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6065	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6066
6067	VMMRZCallRing3Disable(pVCpu);
6068	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6069	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6070	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
6071	VMMRZCallRing3Enable(pVCpu);
6072	}
6073
6074
6075	/**
6076	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6077	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6078	*
6079	* @param pVCpu Pointer to the VMCPU.
6080	*/
6081	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6082	{
6083	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6084	{
6085	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6086	{
6087	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6088	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6089	AssertRC(rc);
6090	}
6091	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6092	}
6093
6094
6095	/**
6096	* Injects any pending events into the guest if the guest is in a state to
6097	* receive them.
6098	*
6099	* @returns VBox status code (informational status codes included).
6100	* @param pVCpu Pointer to the VMCPU.
6101	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6102	* out-of-sync. Make sure to update the required fields
6103	* before using them.
6104	*/
6105	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6106	{
6107	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6108	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6109	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6110	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6111
6112	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6113	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6114	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6115	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6116	Assert(!TRPMHasTrap(pVCpu));
6117
6118	int rc = VINF_SUCCESS;
6119	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6120	{
6121	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6122	bool fInject = true;
6123	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6124	{
6125	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6126	AssertRCReturn(rc, rc);
6127	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6128	if ( fBlockInt
6129	\|\| fBlockSti
6130	\|\| fBlockMovSS)
6131	{
6132	fInject = false;
6133	}
6134	}
6135	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6136	&& ( fBlockMovSS
6137	\|\| fBlockSti))
6138	{
6139	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6140	fInject = false;
6141	}
6142
6143	if (fInject)
6144	{
6145	Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6146	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6147	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6148	AssertRCReturn(rc, rc);
6149	pVCpu->hm.s.Event.fPending = false;
6150
6151	#ifdef VBOX_WITH_STATISTICS
6152	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6153	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6154	else
6155	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6156	#endif
6157	}
6158	else
6159	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6160	} /** @todo SMI. SMIs take priority over NMIs. */
6161	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6162	{
6163	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6164	if ( !fBlockMovSS
6165	&& !fBlockSti)
6166	{
6167	Log4(("Injecting NMI\n"));
6168	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6169	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6170	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6171	0 /* GCPtrFaultAddress */, &uIntrState);
6172	AssertRCReturn(rc, rc);
6173	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6174
6175	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6176	}
6177	else
6178	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6179	}
6180	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6181	{
6182	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6183	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6184	AssertRCReturn(rc, rc);
6185	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6186	if ( !fBlockInt
6187	&& !fBlockSti
6188	&& !fBlockMovSS)
6189	{
6190	uint8_t u8Interrupt;
6191	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6192	if (RT_SUCCESS(rc))
6193	{
6194	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6195	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6196	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6197	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6198	0 /* GCPtrFaultAddress */, &uIntrState);
6199
6200	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6201	}
6202	else
6203	{
6204	/** @todo Does this actually happen? If not turn it into an assertion. */
6205	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6206	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6207	rc = VINF_SUCCESS;
6208	}
6209	}
6210	else
6211	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6212	}
6213
6214	/*
6215	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6216	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6217	*/
6218	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6219	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6220	int rc2 = VINF_SUCCESS;
6221	if ( fBlockSti
6222	\|\| fBlockMovSS)
6223	{
6224	if (!DBGFIsStepping(pVCpu))
6225	{
6226	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6227	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6228	{
6229	/*
6230	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6231	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6232	*/
6233	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6234	AssertRCReturn(rc, rc);
6235	}
6236	}
6237	else
6238	{
6239	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6240	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6241	uIntrState = 0;
6242	}
6243	}
6244
6245	/*
6246	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6247	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6248	*/
6249	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6250	AssertRC(rc2);
6251
6252	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6253	return rc;
6254	}
6255
6256
6257	/**
6258	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6259	*
6260	* @param pVCpu Pointer to the VMCPU.
6261	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6262	* out-of-sync. Make sure to update the required fields
6263	* before using them.
6264	*/
6265	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6266	{
6267	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6268	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6269	}
6270
6271
6272	/**
6273	* Injects a double-fault (#DF) exception into the VM.
6274	*
6275	* @returns VBox status code (informational status code included).
6276	* @param pVCpu Pointer to the VMCPU.
6277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6278	* out-of-sync. Make sure to update the required fields
6279	* before using them.
6280	*/
6281	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6282	{
6283	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6284	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6285	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6286	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6287	puIntrState);
6288	}
6289
6290
6291	/**
6292	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6293	*
6294	* @param pVCpu Pointer to the VMCPU.
6295	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6296	* out-of-sync. Make sure to update the required fields
6297	* before using them.
6298	*/
6299	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6300	{
6301	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6302	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6303	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6304	}
6305
6306
6307	/**
6308	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6309	*
6310	* @param pVCpu Pointer to the VMCPU.
6311	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6312	* out-of-sync. Make sure to update the required fields
6313	* before using them.
6314	* @param cbInstr The value of RIP that is to be pushed on the guest
6315	* stack.
6316	*/
6317	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6318	{
6319	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6320	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6321	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6322	}
6323
6324
6325	/**
6326	* Injects a general-protection (#GP) fault into the VM.
6327	*
6328	* @returns VBox status code (informational status code included).
6329	* @param pVCpu Pointer to the VMCPU.
6330	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6331	* out-of-sync. Make sure to update the required fields
6332	* before using them.
6333	* @param u32ErrorCode The error code associated with the #GP.
6334	*/
6335	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6336	uint32_t *puIntrState)
6337	{
6338	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6339	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6340	if (fErrorCodeValid)
6341	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6342	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6343	puIntrState);
6344	}
6345
6346
6347	/**
6348	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6349	*
6350	* @param pVCpu Pointer to the VMCPU.
6351	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6352	* out-of-sync. Make sure to update the required fields
6353	* before using them.
6354	* @param uVector The software interrupt vector number.
6355	* @param cbInstr The value of RIP that is to be pushed on the guest
6356	* stack.
6357	*/
6358	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6359	{
6360	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6361	if ( uVector == X86_XCPT_BP
6362	\|\| uVector == X86_XCPT_OF)
6363	{
6364	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6365	}
6366	else
6367	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6368	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6369	}
6370
6371
6372	/**
6373	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6374	* stack.
6375	*
6376	* @returns VBox status code (information status code included).
6377	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6378	* @param pVM Pointer to the VM.
6379	* @param pMixedCtx Pointer to the guest-CPU context.
6380	* @param uValue The value to push to the guest stack.
6381	*/
6382	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6383	{
6384	/*
6385	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6386	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6387	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6388	*/
6389	if (pMixedCtx->sp == 1)
6390	return VINF_EM_RESET;
6391	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6392	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6393	AssertRCReturn(rc, rc);
6394	return rc;
6395	}
6396
6397
6398	/**
6399	* Injects an event into the guest upon VM-entry by updating the relevant fields
6400	* in the VM-entry area in the VMCS.
6401	*
6402	* @returns VBox status code (informational error codes included).
6403	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6404	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6405	*
6406	* @param pVCpu Pointer to the VMCPU.
6407	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6408	* be out-of-sync. Make sure to update the required
6409	* fields before using them.
6410	* @param u64IntrInfo The VM-entry interruption-information field.
6411	* @param cbInstr The VM-entry instruction length in bytes (for
6412	* software interrupts, exceptions and privileged
6413	* software exceptions).
6414	* @param u32ErrCode The VM-entry exception error code.
6415	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6416	* @param puIntrState Pointer to the current guest interruptibility-state.
6417	* This interruptibility-state will be updated if
6418	* necessary. This cannot not be NULL.
6419	*
6420	* @remarks No-long-jump zone!!!
6421	* @remarks Requires CR0!
6422	*/
6423	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6424	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6425	{
6426	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6427	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6428	Assert(puIntrState);
6429	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6430
6431	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6432	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6433
6434	#ifdef VBOX_STRICT
6435	/* Validate the error-code-valid bit for hardware exceptions. */
6436	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6437	{
6438	switch (uVector)
6439	{
6440	case X86_XCPT_PF:
6441	case X86_XCPT_DF:
6442	case X86_XCPT_TS:
6443	case X86_XCPT_NP:
6444	case X86_XCPT_SS:
6445	case X86_XCPT_GP:
6446	case X86_XCPT_AC:
6447	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6448	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6449	/* fallthru */
6450	default:
6451	break;
6452	}
6453	}
6454	#endif
6455
6456	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6457	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6458	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6459
6460	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6461
6462	/* We require CR0 to check if the guest is in real-mode. */
6463	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6464	AssertRCReturn(rc, rc);
6465
6466	/*
6467	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6468	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6469	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6470	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6471	*/
6472	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6473	{
6474	PVM pVM = pVCpu->CTX_SUFF(pVM);
6475	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6476	{
6477	Assert(PDMVmmDevHeapIsEnabled(pVM));
6478	Assert(pVM->hm.s.vmx.pRealModeTSS);
6479
6480	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6481	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6482	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6483	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6484	AssertRCReturn(rc, rc);
6485	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6486
6487	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6488	const size_t cbIdtEntry = 4;
6489	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6490	{
6491	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6492	if (uVector == X86_XCPT_DF)
6493	return VINF_EM_RESET;
6494	else if (uVector == X86_XCPT_GP)
6495	{
6496	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6497	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6498	}
6499
6500	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6501	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6502	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6503	}
6504
6505	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6506	uint16_t uGuestIp = pMixedCtx->ip;
6507	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6508	{
6509	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6510	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6511	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6512	}
6513	else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6514	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6515
6516	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6517	uint16_t offIdtEntry = 0;
6518	RTSEL selIdtEntry = 0;
6519	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6520	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6521	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6522	AssertRCReturn(rc, rc);
6523
6524	/* Construct the stack frame for the interrupt/exception handler. */
6525	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6526	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6527	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6528	AssertRCReturn(rc, rc);
6529
6530	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6531	if (rc == VINF_SUCCESS)
6532	{
6533	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6534	pMixedCtx->rip = offIdtEntry;
6535	pMixedCtx->cs.Sel = selIdtEntry;
6536	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6537	if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6538	&& uVector == X86_XCPT_PF)
6539	{
6540	pMixedCtx->cr2 = GCPtrFaultAddress;
6541	}
6542	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6543	\| HM_CHANGED_GUEST_RIP
6544	\| HM_CHANGED_GUEST_RFLAGS
6545	\| HM_CHANGED_GUEST_RSP;
6546
6547	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6548	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6549	{
6550	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6551	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6552	Log4(("Clearing inhibition due to STI.\n"));
6553	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6554	}
6555	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6556	}
6557	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6558	return rc;
6559	}
6560	else
6561	{
6562	/*
6563	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6564	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6565	*/
6566	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6567	}
6568	}
6569
6570	/* Validate. */
6571	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6572	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6573	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6574
6575	/* Inject. */
6576	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6577	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6578	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6579	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6580
6581	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6582	&& uVector == X86_XCPT_PF)
6583	{
6584	pMixedCtx->cr2 = GCPtrFaultAddress;
6585	}
6586
6587	Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6588	u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6589
6590	AssertRCReturn(rc, rc);
6591	return rc;
6592	}
6593
6594
6595	/**
6596	* Enters the VT-x session.
6597	*
6598	* @returns VBox status code.
6599	* @param pVM Pointer to the VM.
6600	* @param pVCpu Pointer to the VMCPU.
6601	* @param pCpu Pointer to the CPU info struct.
6602	*/
6603	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6604	{
6605	AssertPtr(pVM);
6606	AssertPtr(pVCpu);
6607	Assert(pVM->hm.s.vmx.fSupported);
6608	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6609	NOREF(pCpu);
6610
6611	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6612
6613	/* Make sure we're in VMX root mode. */
6614	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6615	if (!(u32HostCR4 & X86_CR4_VMXE))
6616	{
6617	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6618	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6619	}
6620
6621	/* Load the active VMCS as the current one. */
6622	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6623	if (RT_FAILURE(rc))
6624	return rc;
6625
6626	/** @todo this will change with preemption hooks where can can VMRESUME as long
6627	* as we're no preempted. */
6628	pVCpu->hm.s.fResumeVM = false;
6629	return VINF_SUCCESS;
6630	}
6631
6632
6633	/**
6634	* Leaves the VT-x session.
6635	*
6636	* @returns VBox status code.
6637	* @param pVM Pointer to the VM.
6638	* @param pVCpu Pointer to the VMCPU.
6639	* @param pCtx Pointer to the guest-CPU context.
6640	*/
6641	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6642	{
6643	AssertPtr(pVCpu);
6644	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6645	NOREF(pVM);
6646	NOREF(pCtx);
6647
6648	/** @todo this will change with preemption hooks where we only VMCLEAR when
6649	* we are actually going to be preempted, not all the time like we
6650	* currently do. */
6651
6652	/* Restore host-state bits that VT-x only restores partially. */
6653	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6654	{
6655	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6656	/** @todo r=ramshankar: This is broken when
6657	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6658	* VMXRestoreHostState() may unconditionally enables interrupts. */
6659	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6660	#else
6661	Assert(ASMIntAreEnabled());
6662	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6663	#endif
6664	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6665	}
6666
6667	/*
6668	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6669	* and mark the VMCS launch-state as "clear".
6670	*/
6671	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6672	return rc;
6673	}
6674
6675
6676	/**
6677	* Saves the host state in the VMCS host-state.
6678	* Sets up the VM-exit MSR-load area.
6679	*
6680	* The CPU state will be loaded from these fields on every successful VM-exit.
6681	*
6682	* @returns VBox status code.
6683	* @param pVM Pointer to the VM.
6684	* @param pVCpu Pointer to the VMCPU.
6685	*
6686	* @remarks No-long-jump zone!!!
6687	*/
6688	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6689	{
6690	AssertPtr(pVM);
6691	AssertPtr(pVCpu);
6692	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6693
6694	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6695
6696	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6697	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6698	return VINF_SUCCESS;
6699
6700	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6701	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6702
6703	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6704	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6705
6706	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6707	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6708
6709	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6710	return rc;
6711	}
6712
6713
6714	/**
6715	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6716	* loaded from these fields on every successful VM-entry.
6717	*
6718	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6719	* Sets up the VM-entry controls.
6720	* Sets up the appropriate VMX non-root function to execute guest code based on
6721	* the guest CPU mode.
6722	*
6723	* @returns VBox status code.
6724	* @param pVM Pointer to the VM.
6725	* @param pVCpu Pointer to the VMCPU.
6726	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6727	* out-of-sync. Make sure to update the required fields
6728	* before using them.
6729	*
6730	* @remarks No-long-jump zone!!!
6731	*/
6732	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6733	{
6734	AssertPtr(pVM);
6735	AssertPtr(pVCpu);
6736	AssertPtr(pMixedCtx);
6737	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6738
6739	#ifdef LOG_ENABLED
6740	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
6741	* probably not initialized yet? Anyway this will do for now. */
6742	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
6743	VMMR0LogFlushDisable(pVCpu);
6744	#endif
6745
6746	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6747
6748	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6749
6750	/* Determine real-on-v86 mode. */
6751	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6752	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6753	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6754	{
6755	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6756	}
6757
6758	/*
6759	* Load the guest-state into the VMCS.
6760	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6761	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6762	*/
6763	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6764	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6765
6766	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6767	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6768
6769	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6770	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6771
6772	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6773	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6774
6775	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6776	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6777	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6778
6779	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6780	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6781
6782	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6783	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6784
6785	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6786	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6787
6788	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6789	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6790	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6791
6792	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6793	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6794
6795	/* Clear any unused and reserved bits. */
6796	pVCpu->hm.s.fContextUseFlags &= ~( HM_CHANGED_GUEST_CR2
6797	\| HM_CHANGED_GUEST_MSR /* legacy */);
6798
6799	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6800	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
6801	pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
6802
6803	#ifdef LOG_ENABLED
6804	/* Only reenable log-flushing if the caller has it enabled. */
6805	if (!fCallerDisabledLogFlush)
6806	VMMR0LogFlushEnable(pVCpu);
6807	#endif
6808
6809	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6810	return rc;
6811	}
6812
6813
6814	/**
6815	* Loads the guest state into the VMCS guest-state area.
6816	*
6817	* @returns VBox status code.
6818	* @param pVM Pointer to the VM.
6819	* @param pVCpu Pointer to the VMCPU.
6820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6821	* out-of-sync. Make sure to update the required fields
6822	* before using them.
6823	*
6824	* @remarks No-long-jump zone!!!
6825	*/
6826	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6827	{
6828	/*
6829	* Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
6830	* This only helps when we get rescheduled more than once to a different host CPU on a longjmp trip before
6831	* finally executing guest code.
6832	*/
6833	return VINF_SUCCESS;
6834	}
6835
6836
6837	/**
6838	* Does the preparations before executing guest code in VT-x.
6839	*
6840	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6841	* recompiler. We must be cautious what we do here regarding committing
6842	* guest-state information into the the VMCS assuming we assuredly execute the
6843	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6844	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6845	* that the recompiler can (and should) use them when it resumes guest
6846	* execution. Otherwise such operations must be done when we can no longer
6847	* exit to ring-3.
6848	*
6849	* @returns VBox status code (informational status codes included).
6850	* @retval VINF_SUCCESS if we can proceed with running the guest.
6851	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6852	* into the guest.
6853	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6854	*
6855	* @param pVM Pointer to the VM.
6856	* @param pVCpu Pointer to the VMCPU.
6857	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6858	* out-of-sync. Make sure to update the required fields
6859	* before using them.
6860	* @param pVmxTransient Pointer to the VMX transient structure.
6861	*
6862	* @remarks Called with preemption disabled.
6863	*/
6864	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6865	{
6866	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6867
6868	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6869	PGMRZDynMapFlushAutoSet(pVCpu);
6870	#endif
6871
6872	/* Check force flag actions that might require us to go back to ring-3. */
6873	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6874	if (rc != VINF_SUCCESS)
6875	return rc;
6876
6877	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6878	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6879	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6880	{
6881	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6882	RTGCPHYS GCPhysApicBase;
6883	GCPhysApicBase = pMixedCtx->msrApicBase;
6884	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6885
6886	/* Unalias any existing mapping. */
6887	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6888	AssertRCReturn(rc, rc);
6889
6890	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6891	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6892	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6893	AssertRCReturn(rc, rc);
6894
6895	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6896	}
6897
6898	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6899	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6900	pVmxTransient->uEFlags = ASMIntDisableFlags();
6901	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6902	{
6903	ASMSetFlags(pVmxTransient->uEFlags);
6904	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6905	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6906	return VINF_EM_RAW_INTERRUPT;
6907	}
6908	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6909	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6910	#endif
6911
6912	/*
6913	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6914	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6915	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6916	*/
6917	/** @todo Rework event evaluation and injection to be completely separate. */
6918	if (TRPMHasTrap(pVCpu))
6919	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6920
6921	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6922	AssertRCReturn(rc, rc);
6923	return rc;
6924	}
6925
6926
6927	/**
6928	* Prepares to run guest code in VT-x and we've committed to doing so. This
6929	* means there is no backing out to ring-3 or anywhere else at this
6930	* point.
6931	*
6932	* @param pVM Pointer to the VM.
6933	* @param pVCpu Pointer to the VMCPU.
6934	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6935	* out-of-sync. Make sure to update the required fields
6936	* before using them.
6937	* @param pVmxTransient Pointer to the VMX transient structure.
6938	*
6939	* @remarks Called with preemption disabled.
6940	* @remarks No-long-jump zone!!!
6941	*/
6942	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6943	{
6944	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6945	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6946
6947	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6948	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6949	pVmxTransient->uEFlags = ASMIntDisableFlags();
6950	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6951	#endif
6952
6953	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6954	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6955	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6956	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6957	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6958	#endif
6959	int rc = VINF_SUCCESS;
6960	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6961	{
6962	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6963	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6964	}
6965	else if (pVCpu->hm.s.fContextUseFlags)
6966	{
6967	rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
6968	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6969	}
6970	AssertRC(rc);
6971	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6972
6973	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6974	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6975	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6976
6977	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6978	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6979	{
6980	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6981	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6982	}
6983
6984	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6985	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6986	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6987
6988	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6989
6990	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6991	to start executing. */
6992
6993	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6994	/*
6995	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6996	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6997	*/
6998	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6999	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7000	{
7001	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7002	uint64_t u64HostTscAux = 0;
7003	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7004	AssertRC(rc2);
7005	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7006	}
7007	#endif
7008	}
7009
7010
7011	/**
7012	* Performs some essential restoration of state after running guest code in
7013	* VT-x.
7014	*
7015	* @param pVM Pointer to the VM.
7016	* @param pVCpu Pointer to the VMCPU.
7017	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7018	* out-of-sync. Make sure to update the required fields
7019	* before using them.
7020	* @param pVmxTransient Pointer to the VMX transient structure.
7021	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7022	*
7023	* @remarks Called with interrupts disabled.
7024	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7025	* unconditionally when it is safe to do so.
7026	*/
7027	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7028	{
7029	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7030
7031	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7032	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7033	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7034	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7035	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7036
7037	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7038	{
7039	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7040	/* Restore host's TSC_AUX. */
7041	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7042	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7043	#endif
7044	/** @todo Find a way to fix hardcoding a guestimate. */
7045	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7046	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7047	}
7048
7049	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7050	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7051	Assert(!(ASMGetFlags() & X86_EFL_IF));
7052	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7053
7054	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
7055	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7056
7057	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7058	uint32_t uExitReason;
7059	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7060	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7061	AssertRC(rc);
7062	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7063	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7064
7065	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7066	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7067
7068	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7069	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7070	{
7071	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7072	pVmxTransient->fVMEntryFailed));
7073	return;
7074	}
7075
7076	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7077	{
7078	/* Update the guest interruptibility-state from the VMCS. */
7079	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7080	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
7081	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7082	AssertRC(rc);
7083	#endif
7084	/*
7085	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7086	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7087	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7088	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7089	*/
7090	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7091	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7092	{
7093	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7094	AssertRC(rc);
7095	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7096	}
7097	}
7098	}
7099
7100
7101	/**
7102	* Runs the guest code using VT-x.
7103	*
7104	* @returns VBox status code.
7105	* @param pVM Pointer to the VM.
7106	* @param pVCpu Pointer to the VMCPU.
7107	* @param pCtx Pointer to the guest-CPU context.
7108	*
7109	* @remarks Called with preemption disabled.
7110	*/
7111	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7112	{
7113	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7114	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7115
7116	VMXTRANSIENT VmxTransient;
7117	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7118	int rc = VERR_INTERNAL_ERROR_5;
7119	uint32_t cLoops = 0;
7120
7121	for (;; cLoops++)
7122	{
7123	Assert(!HMR0SuspendPending());
7124	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
7125	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
7126	(unsigned)RTMpCpuId(), cLoops));
7127
7128	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7129	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7130	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7131	if (rc != VINF_SUCCESS)
7132	break;
7133
7134	/*
7135	* No longjmps to ring-3 from this point on!!!
7136	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7137	* This also disables flushing of the R0-logger instance (if any).
7138	*/
7139	VMMRZCallRing3Disable(pVCpu);
7140	VMMRZCallRing3RemoveNotification(pVCpu);
7141	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7142
7143	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7144	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7145
7146	/*
7147	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7148	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7149	*/
7150	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7151	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7152	{
7153	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7154	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7155	return rc;
7156	}
7157
7158	/* Handle the VM-exit. */
7159	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7160	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7161	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7162	HMVMX_START_EXIT_DISPATCH_PROF();
7163	#ifdef HMVMX_USE_FUNCTION_TABLE
7164	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7165	#else
7166	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7167	#endif
7168	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7169	if (rc != VINF_SUCCESS)
7170	break;
7171	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7172	{
7173	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7174	rc = VINF_EM_RAW_INTERRUPT;
7175	break;
7176	}
7177	}
7178
7179	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7180	if (rc == VERR_EM_INTERPRETER)
7181	rc = VINF_EM_RAW_EMULATE_INSTR;
7182	else if (rc == VINF_EM_RESET)
7183	rc = VINF_EM_TRIPLE_FAULT;
7184	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7185	return rc;
7186	}
7187
7188
7189	#ifndef HMVMX_USE_FUNCTION_TABLE
7190	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7191	{
7192	int rc;
7193	switch (rcReason)
7194	{
7195	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7196	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7197	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7198	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7199	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7200	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7201	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7202	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7203	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7204	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7205	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7206	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7207	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7208	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7209	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7210	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7211	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7212	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7213	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7214	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7215	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7216	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7217	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7218	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7219	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7220	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7221	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7222	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7223	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7224	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7225	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7226	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7227	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7228
7229	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7230	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7231	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7232	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7233	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7234	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7235	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7236	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7237	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7238
7239	case VMX_EXIT_VMCALL:
7240	case VMX_EXIT_VMCLEAR:
7241	case VMX_EXIT_VMLAUNCH:
7242	case VMX_EXIT_VMPTRLD:
7243	case VMX_EXIT_VMPTRST:
7244	case VMX_EXIT_VMREAD:
7245	case VMX_EXIT_VMRESUME:
7246	case VMX_EXIT_VMWRITE:
7247	case VMX_EXIT_VMXOFF:
7248	case VMX_EXIT_VMXON:
7249	case VMX_EXIT_INVEPT:
7250	case VMX_EXIT_INVVPID:
7251	case VMX_EXIT_VMFUNC:
7252	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7253	break;
7254	default:
7255	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7256	break;
7257	}
7258	return rc;
7259	}
7260	#endif
7261
7262	#ifdef DEBUG
7263	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7264	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7265	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7266
7267	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7268	do \
7269	{ \
7270	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7271	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7272	} while (0)
7273
7274	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7275	do { \
7276	AssertPtr(pVCpu); \
7277	AssertPtr(pMixedCtx); \
7278	AssertPtr(pVmxTransient); \
7279	Assert(pVmxTransient->fVMEntryFailed == false); \
7280	Assert(ASMIntAreEnabled()); \
7281	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7282	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7283	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7284	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7285	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7286	HMVMX_ASSERT_PREEMPT_CPUID(); \
7287	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7288	} while (0)
7289
7290	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7291	do { \
7292	Log4Func(("\n")); \
7293	} while(0)
7294	#else /* Release builds */
7295	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7296	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7297	#endif
7298
7299
7300	/**
7301	* Advances the guest RIP after reading it from the VMCS.
7302	*
7303	* @returns VBox status code.
7304	* @param pVCpu Pointer to the VMCPU.
7305	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7306	* out-of-sync. Make sure to update the required fields
7307	* before using them.
7308	* @param pVmxTransient Pointer to the VMX transient structure.
7309	*
7310	* @remarks No-long-jump zone!!!
7311	*/
7312	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7313	{
7314	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7315	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7316	AssertRCReturn(rc, rc);
7317
7318	pMixedCtx->rip += pVmxTransient->cbInstr;
7319	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7320	return rc;
7321	}
7322
7323
7324	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7325	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7326	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7327
7328	/** @name VM-exit handlers.
7329	* @{
7330	*/
7331
7332	/**
7333	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7334	*/
7335	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7336	{
7337	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7339	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7340	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7341	Assert(ASMIntAreEnabled());
7342	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
7343	return VINF_SUCCESS;
7344	#endif
7345	return VINF_EM_RAW_INTERRUPT;
7346	}
7347
7348
7349	/**
7350	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7351	*/
7352	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7353	{
7354	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7355	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7356
7357	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7358	AssertRCReturn(rc, rc);
7359
7360	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7361	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7362	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7363
7364	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7365	{
7366	/** @todo We should inject the NMI to the host by calling the NMI
7367	* interrupt handler! See Intel spec. 27.5.5 "Updating Non-Register State". */
7368	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7369	return VINF_EM_RAW_INTERRUPT;
7370	}
7371
7372	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7373	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7374	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
7375	{
7376	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7377	return VINF_SUCCESS;
7378	}
7379	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7380	{
7381	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7382	return rc;
7383	}
7384
7385	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7386	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7387	switch (uIntrType)
7388	{
7389	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7390	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7391	/* no break */
7392	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7393	{
7394	switch (uVector)
7395	{
7396	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7397	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7398	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7399	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7400	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7401	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7402	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7403	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7404	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7405	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7406	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7407	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7408	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7409	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7410	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7411	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7412	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7413	#endif
7414	default:
7415	{
7416	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7417	AssertRCReturn(rc, rc);
7418
7419	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7420	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7421	{
7422	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7423	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7424	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7425	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7426	AssertRCReturn(rc, rc);
7427	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7428	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7429	0 /* GCPtrFaultAddress */);
7430	AssertRCReturn(rc, rc);
7431	}
7432	else
7433	{
7434	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7435	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7436	}
7437	break;
7438	}
7439	}
7440	break;
7441	}
7442
7443	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7444	default:
7445	{
7446	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7447	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7448	break;
7449	}
7450	}
7451	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7452	return rc;
7453	}
7454
7455
7456	/**
7457	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7458	*/
7459	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7460	{
7461	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7462
7463	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7464	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7465	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7466	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7467	AssertRCReturn(rc, rc);
7468
7469	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7471	return VINF_SUCCESS;
7472	}
7473
7474
7475	/**
7476	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7477	*/
7478	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7479	{
7480	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7481	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7482	pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
7483	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7484	}
7485
7486
7487	/**
7488	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7489	*/
7490	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7491	{
7492	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7494	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7495	}
7496
7497
7498	/**
7499	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7500	*/
7501	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7502	{
7503	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7504	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7505	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7506	}
7507
7508
7509	/**
7510	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7511	*/
7512	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7513	{
7514	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7515	PVM pVM = pVCpu->CTX_SUFF(pVM);
7516	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7517	if (RT_LIKELY(rc == VINF_SUCCESS))
7518	{
7519	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7520	Assert(pVmxTransient->cbInstr == 2);
7521	}
7522	else
7523	{
7524	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7525	rc = VERR_EM_INTERPRETER;
7526	}
7527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7528	return rc;
7529	}
7530
7531
7532	/**
7533	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7534	*/
7535	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7536	{
7537	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7538	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7539	AssertRCReturn(rc, rc);
7540
7541	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7542	return VINF_EM_RAW_EMULATE_INSTR;
7543
7544	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7545	pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
7546	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7547	}
7548
7549
7550	/**
7551	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7552	*/
7553	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7554	{
7555	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7556	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7557	AssertRCReturn(rc, rc);
7558
7559	PVM pVM = pVCpu->CTX_SUFF(pVM);
7560	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7561	if (RT_LIKELY(rc == VINF_SUCCESS))
7562	{
7563	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7564	Assert(pVmxTransient->cbInstr == 2);
7565	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7566	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7567	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7568	}
7569	else
7570	{
7571	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7572	rc = VERR_EM_INTERPRETER;
7573	}
7574	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7575	return rc;
7576	}
7577
7578
7579	/**
7580	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7581	*/
7582	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7583	{
7584	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7585	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7586	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7587	AssertRCReturn(rc, rc);
7588
7589	PVM pVM = pVCpu->CTX_SUFF(pVM);
7590	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7591	if (RT_LIKELY(rc == VINF_SUCCESS))
7592	{
7593	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7594	Assert(pVmxTransient->cbInstr == 3);
7595	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7596	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7597	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7598	}
7599	else
7600	{
7601	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7602	rc = VERR_EM_INTERPRETER;
7603	}
7604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7605	return rc;
7606	}
7607
7608
7609	/**
7610	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7611	*/
7612	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7613	{
7614	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7615	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7616	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7617	AssertRCReturn(rc, rc);
7618
7619	PVM pVM = pVCpu->CTX_SUFF(pVM);
7620	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7621	if (RT_LIKELY(rc == VINF_SUCCESS))
7622	{
7623	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7624	Assert(pVmxTransient->cbInstr == 2);
7625	}
7626	else
7627	{
7628	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7629	rc = VERR_EM_INTERPRETER;
7630	}
7631	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7632	return rc;
7633	}
7634
7635
7636	/**
7637	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7638	*/
7639	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7640	{
7641	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7642	PVM pVM = pVCpu->CTX_SUFF(pVM);
7643	Assert(!pVM->hm.s.fNestedPaging);
7644
7645	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7646	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7647	AssertRCReturn(rc, rc);
7648
7649	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7650	rc = VBOXSTRICTRC_VAL(rc2);
7651	if (RT_LIKELY(rc == VINF_SUCCESS))
7652	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7653	else
7654	{
7655	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7656	pVmxTransient->uExitQualification, rc));
7657	}
7658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7659	return rc;
7660	}
7661
7662
7663	/**
7664	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7665	*/
7666	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7667	{
7668	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7669	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7670	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7671	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7672	AssertRCReturn(rc, rc);
7673
7674	PVM pVM = pVCpu->CTX_SUFF(pVM);
7675	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7676	if (RT_LIKELY(rc == VINF_SUCCESS))
7677	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7678	else
7679	{
7680	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7681	rc = VERR_EM_INTERPRETER;
7682	}
7683	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7684	return rc;
7685	}
7686
7687
7688	/**
7689	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7690	*/
7691	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7692	{
7693	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7694	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7695	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7696	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7697	AssertRCReturn(rc, rc);
7698
7699	PVM pVM = pVCpu->CTX_SUFF(pVM);
7700	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7701	rc = VBOXSTRICTRC_VAL(rc2);
7702	if (RT_LIKELY( rc == VINF_SUCCESS
7703	\|\| rc == VINF_EM_HALT))
7704	{
7705	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7706	AssertRCReturn(rc3, rc3);
7707
7708	if ( rc == VINF_EM_HALT
7709	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7710	{
7711	rc = VINF_SUCCESS;
7712	}
7713	}
7714	else
7715	{
7716	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7717	rc = VERR_EM_INTERPRETER;
7718	}
7719	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7720	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7721	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7722	return rc;
7723	}
7724
7725
7726	/**
7727	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7728	*/
7729	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7730	{
7731	/*
7732	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7733	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7734	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
7735	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7736	*/
7737	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7738	pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
7739	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7740	}
7741
7742
7743	/**
7744	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7745	*/
7746	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7747	{
7748	/*
7749	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7750	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
7751	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
7752	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7753	*/
7754	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7755	pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
7756	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7757	}
7758
7759
7760	/**
7761	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7762	*/
7763	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7764	{
7765	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7766	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7767	pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
7768	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7769	}
7770
7771
7772	/**
7773	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7774	*/
7775	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7776	{
7777	/*
7778	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7779	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7780	* See Intel spec. 25.3 "Other Causes of VM-exits".
7781	*/
7782	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7783	pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
7784	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7785	}
7786
7787
7788	/**
7789	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7790	* VM-exit.
7791	*/
7792	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7793	{
7794	/*
7795	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7796	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7797	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7798	*/
7799	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7800	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7801	}
7802
7803
7804	/**
7805	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7806	* VM-exit.
7807	*/
7808	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7809	{
7810	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7811	return VINF_EM_RESET;
7812	}
7813
7814
7815	/**
7816	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7817	*/
7818	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7819	{
7820	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7821	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7822	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7823	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7824	AssertRCReturn(rc, rc);
7825
7826	pMixedCtx->rip++;
7827	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7828	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7829	rc = VINF_SUCCESS;
7830	else
7831	rc = VINF_EM_HALT;
7832
7833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7834	return rc;
7835	}
7836
7837
7838	/**
7839	* VM-exit handler for instructions that result in a #UD exception delivered to
7840	* the guest.
7841	*/
7842	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7843	{
7844	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7845	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7846	return VINF_SUCCESS;
7847	}
7848
7849
7850	/**
7851	* VM-exit handler for expiry of the VMX preemption timer.
7852	*/
7853	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7854	{
7855	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7856
7857	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7858	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7859
7860	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7861	PVM pVM = pVCpu->CTX_SUFF(pVM);
7862	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7864	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7865	}
7866
7867
7868	/**
7869	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7870	*/
7871	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7872	{
7873	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7874
7875	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7876	/** @todo check if XSETBV is supported by the recompiler. */
7877	return VERR_EM_INTERPRETER;
7878	}
7879
7880
7881	/**
7882	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7883	*/
7884	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7885	{
7886	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7887
7888	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7889	/** @todo implement EMInterpretInvpcid() */
7890	return VERR_EM_INTERPRETER;
7891	}
7892
7893
7894	/**
7895	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7896	* Error VM-exit.
7897	*/
7898	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7899	{
7900	uint32_t uIntrState;
7901	HMVMXHCUINTREG uHCReg;
7902	uint64_t u64Val;
7903	uint32_t u32Val;
7904
7905	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7906	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7907	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7908	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7909	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7910	AssertRCReturn(rc, rc);
7911
7912	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7913	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7914	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7915	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7916
7917	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7918	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7919	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7920	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7921	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7922	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7923	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7924	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7925	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7926	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7927	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7928	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7929
7930	PVM pVM = pVCpu->CTX_SUFF(pVM);
7931	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7932
7933	return VERR_VMX_INVALID_GUEST_STATE;
7934	}
7935
7936
7937	/**
7938	* VM-exit handler for VM-entry failure due to an MSR-load
7939	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7940	*/
7941	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7942	{
7943	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7944	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7945	}
7946
7947
7948	/**
7949	* VM-exit handler for VM-entry failure due to a machine-check event
7950	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7951	*/
7952	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7953	{
7954	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7955	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7956	}
7957
7958
7959	/**
7960	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7961	* theory.
7962	*/
7963	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7964	{
7965	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7966	return VERR_VMX_UNDEFINED_EXIT_CODE;
7967	}
7968
7969
7970	/**
7971	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7972	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7973	* Conditional VM-exit.
7974	*/
7975	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7976	{
7977	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7978
7979	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7981	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7982	return VERR_EM_INTERPRETER;
7983	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7984	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7985	}
7986
7987
7988	/**
7989	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7990	*/
7991	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7992	{
7993	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7994
7995	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7996	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7997	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7998	return VERR_EM_INTERPRETER;
7999	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8000	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8001	}
8002
8003
8004	/**
8005	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
8006	*/
8007	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8008	{
8009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8010
8011	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
8012	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8013	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8014	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8015	AssertRCReturn(rc, rc);
8016
8017	PVM pVM = pVCpu->CTX_SUFF(pVM);
8018	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8019	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
8020	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
8021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
8022
8023	if (RT_LIKELY(rc == VINF_SUCCESS))
8024	{
8025	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8026	Assert(pVmxTransient->cbInstr == 2);
8027	}
8028	return rc;
8029	}
8030
8031
8032	/**
8033	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
8034	*/
8035	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8036	{
8037	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8038	PVM pVM = pVCpu->CTX_SUFF(pVM);
8039	int rc = VINF_SUCCESS;
8040
8041	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
8042	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8043	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8044	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8045	AssertRCReturn(rc, rc);
8046	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
8047
8048	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8049	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
8050	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
8051
8052	if (RT_LIKELY(rc == VINF_SUCCESS))
8053	{
8054	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8055
8056	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
8057	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
8058	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
8059	{
8060	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
8061	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
8062	EMInterpretWrmsr() changes it. */
8063	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8064	}
8065	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
8066	{
8067	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8068	AssertRCReturn(rc, rc);
8069	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
8070	}
8071	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
8072	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8073
8074	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
8075	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8076	{
8077	switch (pMixedCtx->ecx)
8078	{
8079	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
8080	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
8081	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
8082	case MSR_K8_FS_BASE: /* no break */
8083	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
8084	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
8085	}
8086	}
8087	#ifdef VBOX_STRICT
8088	else
8089	{
8090	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
8091	switch (pMixedCtx->ecx)
8092	{
8093	case MSR_IA32_SYSENTER_CS:
8094	case MSR_IA32_SYSENTER_EIP:
8095	case MSR_IA32_SYSENTER_ESP:
8096	case MSR_K8_FS_BASE:
8097	case MSR_K8_GS_BASE:
8098	{
8099	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
8100	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8101	}
8102
8103	case MSR_K8_LSTAR:
8104	case MSR_K6_STAR:
8105	case MSR_K8_SF_MASK:
8106	case MSR_K8_TSC_AUX:
8107	case MSR_K8_KERNEL_GS_BASE:
8108	{
8109	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
8110	pMixedCtx->ecx));
8111	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8112	}
8113	}
8114	}
8115	#endif /* VBOX_STRICT */
8116	}
8117	return rc;
8118	}
8119
8120
8121	/**
8122	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
8123	*/
8124	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8125	{
8126	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8127
8128	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
8129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
8130	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
8131	return VERR_EM_INTERPRETER;
8132	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8133	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8134	}
8135
8136
8137	/**
8138	* VM-exit handler for when the TPR value is lowered below the specified
8139	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
8140	*/
8141	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8142	{
8143	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8144	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
8145
8146	/*
8147	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
8148	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
8149	* resume guest execution.
8150	*/
8151	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
8153	return VINF_SUCCESS;
8154	}
8155
8156
8157	/**
8158	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
8159	* VM-exit.
8160	*
8161	* @retval VINF_SUCCESS when guest execution can continue.
8162	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8163	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8164	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8165	* recompiler.
8166	*/
8167	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8168	{
8169	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8170	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8171	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8172	AssertRCReturn(rc, rc);
8173
8174	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8175	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8176	PVM pVM = pVCpu->CTX_SUFF(pVM);
8177	switch (uAccessType)
8178	{
8179	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8180	{
8181	#if 0
8182	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8183	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8184	#else
8185	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8186	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8187	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8188	#endif
8189	AssertRCReturn(rc, rc);
8190
8191	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8192	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8193	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8194	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8195
8196	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8197	{
8198	case 0: /* CR0 */
8199	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8200	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8201	break;
8202	case 2: /* C2 **/
8203	/* Nothing to do here, CR2 it's not part of the VMCS. */
8204	break;
8205	case 3: /* CR3 */
8206	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8207	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8208	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8209	break;
8210	case 4: /* CR4 */
8211	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8212	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8213	break;
8214	case 8: /* CR8 */
8215	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8216	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8217	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8218	break;
8219	default:
8220	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8221	break;
8222	}
8223
8224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8225	break;
8226	}
8227
8228	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8229	{
8230	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8231	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8232	AssertRCReturn(rc, rc);
8233	Assert( !pVM->hm.s.fNestedPaging
8234	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8235	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8236
8237	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8238	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8239	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8240
8241	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8242	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8243	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8244	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8246	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8247	break;
8248	}
8249
8250	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8251	{
8252	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8253	AssertRCReturn(rc, rc);
8254	rc = EMInterpretCLTS(pVM, pVCpu);
8255	AssertRCReturn(rc, rc);
8256	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8258	Log4(("CRX CLTS write rc=%d\n", rc));
8259	break;
8260	}
8261
8262	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8263	{
8264	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8265	AssertRCReturn(rc, rc);
8266	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8267	if (RT_LIKELY(rc == VINF_SUCCESS))
8268	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8270	Log4(("CRX LMSW write rc=%d\n", rc));
8271	break;
8272	}
8273
8274	default:
8275	{
8276	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8277	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8278	}
8279	}
8280
8281	/* Validate possible error codes. */
8282	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8283	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8284	if (RT_SUCCESS(rc))
8285	{
8286	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8287	AssertRCReturn(rc2, rc2);
8288	}
8289
8290	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8291	return rc;
8292	}
8293
8294
8295	/**
8296	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8297	* VM-exit.
8298	*/
8299	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8300	{
8301	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8302	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8303
8304	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8305	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8306	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8307	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8308	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8309	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8310	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8311	AssertRCReturn(rc, rc);
8312
8313	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8314
8315	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8316	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8317	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8318	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8319	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8320	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8321	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8322
8323	/* I/O operation lookup arrays. */
8324	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
8325	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8326
8327	const uint32_t cbSize = s_aIOSize[uIOWidth];
8328	const uint32_t cbInstr = pVmxTransient->cbInstr;
8329	PVM pVM = pVCpu->CTX_SUFF(pVM);
8330	if (fIOString)
8331	{
8332	/* INS/OUTS - I/O String instruction. */
8333	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8334	/** @todo for now manually disassemble later optimize by getting the fields from
8335	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8336	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8337	* segment prefix info. */
8338	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8339	if (RT_SUCCESS(rc))
8340	{
8341	if (fIOWrite)
8342	{
8343	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8344	(DISCPUMODE)pDis->uAddrMode, cbSize);
8345	rc = VBOXSTRICTRC_VAL(rc2);
8346	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8347	}
8348	else
8349	{
8350	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8351	(DISCPUMODE)pDis->uAddrMode, cbSize);
8352	rc = VBOXSTRICTRC_VAL(rc2);
8353	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8354	}
8355	}
8356	else
8357	{
8358	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8359	rc = VINF_EM_RAW_EMULATE_INSTR;
8360	}
8361	}
8362	else
8363	{
8364	/* IN/OUT - I/O instruction. */
8365	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8366	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8367	if (fIOWrite)
8368	{
8369	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8370	rc = VBOXSTRICTRC_VAL(rc2);
8371	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8372	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8374	}
8375	else
8376	{
8377	uint32_t u32Result = 0;
8378	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8379	rc = VBOXSTRICTRC_VAL(rc2);
8380	if (IOM_SUCCESS(rc))
8381	{
8382	/* Save result of I/O IN instr. in AL/AX/EAX. */
8383	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8384	}
8385	else if (rc == VINF_IOM_R3_IOPORT_READ)
8386	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8388	}
8389	}
8390
8391	if (IOM_SUCCESS(rc))
8392	{
8393	pMixedCtx->rip += cbInstr;
8394	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8395	if (RT_LIKELY(rc == VINF_SUCCESS))
8396	{
8397	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8398	AssertRCReturn(rc, rc);
8399
8400	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8401	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8402	{
8403	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8404	for (unsigned i = 0; i < 4; i++)
8405	{
8406	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8407	if ( ( uIOPort >= pMixedCtx->dr[i]
8408	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8409	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8410	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8411	{
8412	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8413	uint64_t uDR6 = ASMGetDR6();
8414
8415	/* Clear all breakpoint status flags and set the one we just hit. */
8416	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8417	uDR6 \|= (uint64_t)RT_BIT(i);
8418
8419	/*
8420	* Note: AMD64 Architecture Programmer's Manual 13.1:
8421	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8422	* be cleared by software after the contents have been read.
8423	*/
8424	ASMSetDR6(uDR6);
8425
8426	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8427	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8428
8429	/* Paranoia. */
8430	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8431	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8432	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8433
8434	/* Resync DR7 */
8435	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8436	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8437
8438	/* Set #DB to be injected into the VM and continue guest execution. */
8439	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8440	break;
8441	}
8442	}
8443	}
8444	}
8445	}
8446
8447	#ifdef DEBUG
8448	if (rc == VINF_IOM_R3_IOPORT_READ)
8449	Assert(!fIOWrite);
8450	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8451	Assert(fIOWrite);
8452	else
8453	{
8454	AssertMsg( RT_FAILURE(rc)
8455	\|\| rc == VINF_SUCCESS
8456	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8457	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8458	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8459	}
8460	#endif
8461
8462	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8463	return rc;
8464	}
8465
8466
8467	/**
8468	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8469	* VM-exit.
8470	*/
8471	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8472	{
8473	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8474
8475	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8476	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8477	AssertRCReturn(rc, rc);
8478	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8479	{
8480	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8481	AssertRCReturn(rc, rc);
8482	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8483	{
8484	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8485
8486	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8487	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8488	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8489	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8490	{
8491	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8492	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8493
8494	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8495	Assert(!pVCpu->hm.s.Event.fPending);
8496	pVCpu->hm.s.Event.fPending = true;
8497	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8498	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8499	AssertRCReturn(rc, rc);
8500	if (fErrorCodeValid)
8501	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8502	else
8503	pVCpu->hm.s.Event.u32ErrCode = 0;
8504	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8505	&& uVector == X86_XCPT_PF)
8506	{
8507	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8508	}
8509
8510	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8511	}
8512	}
8513	}
8514
8515	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8516	* emulation. */
8517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8518	return VERR_EM_INTERPRETER;
8519	}
8520
8521
8522	/**
8523	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8524	*/
8525	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8526	{
8527	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8528	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8529	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8530	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8531	AssertRCReturn(rc, rc);
8532	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8533	return VINF_EM_DBG_STOP;
8534	}
8535
8536
8537	/**
8538	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8539	*/
8540	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8541	{
8542	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8543
8544	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8545	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8546	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8547	return VINF_SUCCESS;
8548	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8549	return rc;
8550
8551	#if 0
8552	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8553	* just sync the whole thing. */
8554	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8555	#else
8556	/* Aggressive state sync. for now. */
8557	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8558	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8559	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8560	#endif
8561	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8562	AssertRCReturn(rc, rc);
8563
8564	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8565	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8566	switch (uAccessType)
8567	{
8568	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8569	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8570	{
8571	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8572	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8573	{
8574	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8575	}
8576
8577	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8578	GCPhys &= PAGE_BASE_GC_MASK;
8579	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8580	PVM pVM = pVCpu->CTX_SUFF(pVM);
8581	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8582	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8583
8584	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8585	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8586	CPUMCTX2CORE(pMixedCtx), GCPhys);
8587	rc = VBOXSTRICTRC_VAL(rc2);
8588	Log4(("ApicAccess rc=%d\n", rc));
8589	if ( rc == VINF_SUCCESS
8590	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8591	\|\| rc == VERR_PAGE_NOT_PRESENT)
8592	{
8593	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8594	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8595	rc = VINF_SUCCESS;
8596	}
8597	break;
8598	}
8599
8600	default:
8601	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8602	rc = VINF_EM_RAW_EMULATE_INSTR;
8603	break;
8604	}
8605
8606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8607	return rc;
8608	}
8609
8610
8611	/**
8612	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8613	* VM-exit.
8614	*/
8615	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8616	{
8617	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8618
8619	/* We should -not- get this VM-exit if the guest is debugging. */
8620	if (CPUMIsGuestDebugStateActive(pVCpu))
8621	{
8622	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8623	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8624	}
8625
8626	int rc = VERR_INTERNAL_ERROR_5;
8627	if ( !DBGFIsStepping(pVCpu)
8628	&& !CPUMIsHyperDebugStateActive(pVCpu))
8629	{
8630	/* Don't intercept MOV DRx. */
8631	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8632	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8633	AssertRCReturn(rc, rc);
8634
8635	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8636	PVM pVM = pVCpu->CTX_SUFF(pVM);
8637	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8638	AssertRC(rc);
8639	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8640
8641	#ifdef VBOX_WITH_STATISTICS
8642	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8643	AssertRCReturn(rc, rc);
8644	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8645	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8646	else
8647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8648	#endif
8649	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8650	return VINF_SUCCESS;
8651	}
8652
8653	/*
8654	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8655	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8656	*/
8657	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8658	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8659	AssertRCReturn(rc, rc);
8660
8661	PVM pVM = pVCpu->CTX_SUFF(pVM);
8662	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8663	{
8664	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8665	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8666	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8667	if (RT_SUCCESS(rc))
8668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8670	}
8671	else
8672	{
8673	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8674	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8675	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8676	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8677	}
8678
8679	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8680	if (RT_SUCCESS(rc))
8681	{
8682	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8683	AssertRCReturn(rc2, rc2);
8684	}
8685	return rc;
8686	}
8687
8688
8689	/**
8690	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8691	* Conditional VM-exit.
8692	*/
8693	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8694	{
8695	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8696	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8697
8698	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8699	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8700	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8701	return VINF_SUCCESS;
8702	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8703	return rc;
8704
8705	RTGCPHYS GCPhys = 0;
8706	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8707
8708	#if 0
8709	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8710	#else
8711	/* Aggressive state sync. for now. */
8712	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8713	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8714	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8715	#endif
8716	AssertRCReturn(rc, rc);
8717
8718	/*
8719	* If we succeed, resume guest execution.
8720	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8721	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8722	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8723	* weird case. See @bugref{6043}.
8724	*/
8725	PVM pVM = pVCpu->CTX_SUFF(pVM);
8726	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8727	rc = VBOXSTRICTRC_VAL(rc2);
8728	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8729	if ( rc == VINF_SUCCESS
8730	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8731	\|\| rc == VERR_PAGE_NOT_PRESENT)
8732	{
8733	/* Successfully handled MMIO operation. */
8734	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8735	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8736	rc = VINF_SUCCESS;
8737	}
8738	return rc;
8739	}
8740
8741
8742	/**
8743	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8744	* VM-exit.
8745	*/
8746	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8747	{
8748	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8749	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8750
8751	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8752	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8753	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8754	return VINF_SUCCESS;
8755	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8756	return rc;
8757
8758	RTGCPHYS GCPhys = 0;
8759	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8760	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8761	#if 0
8762	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8763	#else
8764	/* Aggressive state sync. for now. */
8765	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8766	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8767	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8768	#endif
8769	AssertRCReturn(rc, rc);
8770
8771	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8772	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8773
8774	RTGCUINT uErrorCode = 0;
8775	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8776	uErrorCode \|= X86_TRAP_PF_ID;
8777	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8778	uErrorCode \|= X86_TRAP_PF_RW;
8779	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8780	uErrorCode \|= X86_TRAP_PF_P;
8781
8782	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8783
8784	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8785	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8786
8787	/* Handle the pagefault trap for the nested shadow table. */
8788	PVM pVM = pVCpu->CTX_SUFF(pVM);
8789	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8790	TRPMResetTrap(pVCpu);
8791
8792	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8793	if ( rc == VINF_SUCCESS
8794	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8795	\|\| rc == VERR_PAGE_NOT_PRESENT)
8796	{
8797	/* Successfully synced our nested page tables. */
8798	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8799	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8800	return VINF_SUCCESS;
8801	}
8802
8803	Log4(("EPT return to ring-3 rc=%d\n"));
8804	return rc;
8805	}
8806
8807	/** @} */
8808
8809	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8810	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8811	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8812
8813	/** @name VM-exit exception handlers.
8814	* @{
8815	*/
8816
8817	/**
8818	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8819	*/
8820	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8821	{
8822	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8823	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8824
8825	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8826	AssertRCReturn(rc, rc);
8827
8828	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8829	{
8830	/* Old-style FPU error reporting needs some extra work. */
8831	/** @todo don't fall back to the recompiler, but do it manually. */
8832	return VERR_EM_INTERPRETER;
8833	}
8834
8835	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8836	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8837	return rc;
8838	}
8839
8840
8841	/**
8842	* VM-exit exception handler for #BP (Breakpoint exception).
8843	*/
8844	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8845	{
8846	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8848
8849	/** @todo Try optimize this by not saving the entire guest state unless
8850	* really needed. */
8851	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8852	AssertRCReturn(rc, rc);
8853
8854	PVM pVM = pVCpu->CTX_SUFF(pVM);
8855	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8856	if (rc == VINF_EM_RAW_GUEST_TRAP)
8857	{
8858	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8859	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8860	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8861	AssertRCReturn(rc, rc);
8862
8863	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8864	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8865	}
8866
8867	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8868	return rc;
8869	}
8870
8871
8872	/**
8873	* VM-exit exception handler for #DB (Debug exception).
8874	*/
8875	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8876	{
8877	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8879
8880	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8881	AssertRCReturn(rc, rc);
8882
8883	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8884	uint64_t uDR6 = X86_DR6_INIT_VAL;
8885	uDR6 \|= (pVmxTransient->uExitQualification
8886	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8887	PVM pVM = pVCpu->CTX_SUFF(pVM);
8888	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8889	if (rc == VINF_EM_RAW_GUEST_TRAP)
8890	{
8891	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8892	pMixedCtx->dr[6] = uDR6;
8893
8894	if (CPUMIsGuestDebugStateActive(pVCpu))
8895	ASMSetDR6(pMixedCtx->dr[6]);
8896
8897	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8898
8899	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8900	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8901
8902	/* Paranoia. */
8903	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8904	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8905	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8906
8907	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8908	AssertRCReturn(rc,rc);
8909
8910	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8911	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8912	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8913	AssertRCReturn(rc2, rc2);
8914	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8915	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8916	rc = VINF_SUCCESS;
8917	}
8918
8919	return rc;
8920	}
8921
8922
8923	/**
8924	* VM-exit exception handler for #NM (Device-not-available exception: floating
8925	* point exception).
8926	*/
8927	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8928	{
8929	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8930
8931	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8932	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8933	#endif
8934
8935	/* We require CR0 and EFER. EFER is always up-to-date. */
8936	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8937	AssertRCReturn(rc, rc);
8938
8939	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8940	PVM pVM = pVCpu->CTX_SUFF(pVM);
8941	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8942	if (rc == VINF_SUCCESS)
8943	{
8944	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8945	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8947	return VINF_SUCCESS;
8948	}
8949
8950	/* Forward #NM to the guest. */
8951	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8952	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8953	AssertRCReturn(rc, rc);
8954	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8955	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8957	return rc;
8958	}
8959
8960
8961	/**
8962	* VM-exit exception handler for #GP (General-protection exception).
8963	*
8964	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8965	*/
8966	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8967	{
8968	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8970
8971	int rc = VERR_INTERNAL_ERROR_5;
8972	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8973	{
8974	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8975	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8976	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8977	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8978	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8979	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8980	AssertRCReturn(rc, rc);
8981	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8982	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8983	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8984	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8985	return rc;
8986	#else
8987	/* We don't intercept #GP. */
8988	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8989	return VERR_VMX_UNEXPECTED_EXCEPTION;
8990	#endif
8991	}
8992
8993	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8994	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8995
8996	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8997	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8998	AssertRCReturn(rc, rc);
8999
9000	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9001	uint32_t cbOp = 0;
9002	PVM pVM = pVCpu->CTX_SUFF(pVM);
9003	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
9004	if (RT_SUCCESS(rc))
9005	{
9006	rc = VINF_SUCCESS;
9007	Assert(cbOp == pDis->cbInstr);
9008	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9009	switch (pDis->pCurInstr->uOpcode)
9010	{
9011	case OP_CLI:
9012	{
9013	pMixedCtx->eflags.Bits.u1IF = 0;
9014	pMixedCtx->rip += pDis->cbInstr;
9015	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9016	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
9017	break;
9018	}
9019
9020	case OP_STI:
9021	{
9022	pMixedCtx->eflags.Bits.u1IF = 1;
9023	pMixedCtx->rip += pDis->cbInstr;
9024	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
9025	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
9026	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
9027	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
9028	break;
9029	}
9030
9031	case OP_HLT:
9032	{
9033	rc = VINF_EM_HALT;
9034	pMixedCtx->rip += pDis->cbInstr;
9035	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
9036	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9037	break;
9038	}
9039
9040	case OP_POPF:
9041	{
9042	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9043	uint32_t cbParm = 0;
9044	uint32_t uMask = 0;
9045	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9046	{
9047	cbParm = 4;
9048	uMask = 0xffffffff;
9049	}
9050	else
9051	{
9052	cbParm = 2;
9053	uMask = 0xffff;
9054	}
9055
9056	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
9057	RTGCPTR GCPtrStack = 0;
9058	X86EFLAGS uEflags;
9059	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9060	&GCPtrStack);
9061	if (RT_SUCCESS(rc))
9062	{
9063	Assert(sizeof(uEflags.u32) >= cbParm);
9064	uEflags.u32 = 0;
9065	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
9066	}
9067	if (RT_FAILURE(rc))
9068	{
9069	rc = VERR_EM_INTERPRETER;
9070	break;
9071	}
9072	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
9073	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9074	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
9075	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
9076	pMixedCtx->eflags.Bits.u1RF = 0;
9077	pMixedCtx->esp += cbParm;
9078	pMixedCtx->esp &= uMask;
9079	pMixedCtx->rip += pDis->cbInstr;
9080	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
9081	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
9082	break;
9083	}
9084
9085	case OP_PUSHF:
9086	{
9087	uint32_t cbParm = 0;
9088	uint32_t uMask = 0;
9089	if (pDis->fPrefix & DISPREFIX_OPSIZE)
9090	{
9091	cbParm = 4;
9092	uMask = 0xffffffff;
9093	}
9094	else
9095	{
9096	cbParm = 2;
9097	uMask = 0xffff;
9098	}
9099
9100	/* Get the stack pointer & push the contents of eflags onto the stack. */
9101	RTGCPTR GCPtrStack = 0;
9102	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
9103	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
9104	if (RT_FAILURE(rc))
9105	{
9106	rc = VERR_EM_INTERPRETER;
9107	break;
9108	}
9109	X86EFLAGS uEflags;
9110	uEflags = pMixedCtx->eflags;
9111	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
9112	uEflags.Bits.u1RF = 0;
9113	uEflags.Bits.u1VM = 0;
9114
9115	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
9116	if (RT_FAILURE(rc))
9117	{
9118	rc = VERR_EM_INTERPRETER;
9119	break;
9120	}
9121	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
9122	pMixedCtx->esp -= cbParm;
9123	pMixedCtx->esp &= uMask;
9124	pMixedCtx->rip += pDis->cbInstr;
9125	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
9126	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
9127	break;
9128	}
9129
9130	case OP_IRET:
9131	{
9132	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
9133	* instruction reference. */
9134	RTGCPTR GCPtrStack = 0;
9135	uint32_t uMask = 0xffff;
9136	uint16_t aIretFrame[3];
9137	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
9138	{
9139	rc = VERR_EM_INTERPRETER;
9140	break;
9141	}
9142	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
9143	&GCPtrStack);
9144	if (RT_SUCCESS(rc))
9145	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
9146	if (RT_FAILURE(rc))
9147	{
9148	rc = VERR_EM_INTERPRETER;
9149	break;
9150	}
9151	pMixedCtx->eip = 0;
9152	pMixedCtx->ip = aIretFrame[0];
9153	pMixedCtx->cs.Sel = aIretFrame[1];
9154	pMixedCtx->cs.ValidSel = aIretFrame[1];
9155	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
9156	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
9157	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
9158	pMixedCtx->sp += sizeof(aIretFrame);
9159	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
9160	\| HM_CHANGED_GUEST_RFLAGS;
9161	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
9162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
9163	break;
9164	}
9165
9166	case OP_INT:
9167	{
9168	uint16_t uVector = pDis->Param1.uValue & 0xff;
9169	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9170	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9171	break;
9172	}
9173
9174	case OP_INTO:
9175	{
9176	if (pMixedCtx->eflags.Bits.u1OF)
9177	{
9178	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9180	}
9181	break;
9182	}
9183
9184	default:
9185	{
9186	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9187	EMCODETYPE_SUPERVISOR);
9188	rc = VBOXSTRICTRC_VAL(rc2);
9189	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9190	Log4(("#GP rc=%Rrc\n", rc));
9191	break;
9192	}
9193	}
9194	}
9195	else
9196	rc = VERR_EM_INTERPRETER;
9197
9198	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9199	("#GP Unexpected rc=%Rrc\n", rc));
9200	return rc;
9201	}
9202
9203
9204	/**
9205	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9206	* the exception reported in the VMX transient structure back into the VM.
9207	*
9208	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9209	* up-to-date.
9210	*/
9211	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9212	{
9213	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9214
9215	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9216	hmR0VmxCheckExitDueToEventDelivery(). */
9217	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9218	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9219	AssertRCReturn(rc, rc);
9220	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9221
9222	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9223	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9224	return VINF_SUCCESS;
9225	}
9226
9227
9228	/**
9229	* VM-exit exception handler for #PF (Page-fault exception).
9230	*/
9231	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9232	{
9233	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9234	PVM pVM = pVCpu->CTX_SUFF(pVM);
9235	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9236	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9237	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9238	AssertRCReturn(rc, rc);
9239
9240	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9241	if (pVM->hm.s.fNestedPaging)
9242	{
9243	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
9244	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9245	{
9246	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9247	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9248	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9249	}
9250	else
9251	{
9252	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9253	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9254	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9255	}
9256	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9257	return rc;
9258	}
9259	#else
9260	Assert(!pVM->hm.s.fNestedPaging);
9261	#endif
9262
9263	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9264	AssertRCReturn(rc, rc);
9265
9266	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9267	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9268
9269	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9270	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9271	(RTGCPTR)pVmxTransient->uExitQualification);
9272
9273	Log4(("#PF: rc=%Rrc\n", rc));
9274	if (rc == VINF_SUCCESS)
9275	{
9276	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9277	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9278	* memory? We don't update the whole state here... */
9279	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9280	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9281	TRPMResetTrap(pVCpu);
9282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9283	return rc;
9284	}
9285	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9286	{
9287	if (!pVmxTransient->fVectoringPF)
9288	{
9289	/* It's a guest page fault and needs to be reflected to the guest. */
9290	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9291	TRPMResetTrap(pVCpu);
9292	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9293	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9294	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9295	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9296	}
9297	else
9298	{
9299	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9300	TRPMResetTrap(pVCpu);
9301	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
9302	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9303	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9304	}
9305
9306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9307	return VINF_SUCCESS;
9308	}
9309
9310	TRPMResetTrap(pVCpu);
9311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9312	return rc;
9313	}
9314
9315	/** @} */
9316

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 47105

Download in other formats: