HMVMXR0.cpp@ 46130

Last change on this file since 46130 was 46103, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Log CS during loading the guest state.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.7 KB

Line
1	/* $Id: HMVMXR0.cpp 46103 2013-05-15 17:02:47Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147	/** Profiling macro. */
148	#ifdef HM_PROFILE_EXIT_DISPATCH
149	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
150	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
151	#else
152	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
153	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
154	#endif
155
156
157	/*******************************************************************************
158	* Structures and Typedefs *
159	*******************************************************************************/
160	/**
161	* A state structure for holding miscellaneous information across
162	* VMX non-root operation and restored after the transition.
163	*/
164	typedef struct VMXTRANSIENT
165	{
166	/** The host's rflags/eflags. */
167	RTCCUINTREG uEFlags;
168	#if HC_ARCH_BITS == 32
169	uint32_t u32Alignment0;
170	#endif
171	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
172	uint64_t u64LStarMsr;
173	/** The guest's TPR value used for TPR shadowing. */
174	uint8_t u8GuestTpr;
175	/** Alignment. */
176	uint8_t abAlignment0[6];
177
178	/** The basic VM-exit reason. */
179	uint16_t uExitReason;
180	/** Alignment. */
181	uint16_t u16Alignment0;
182	/** The VM-exit interruption error code. */
183	uint32_t uExitIntrErrorCode;
184	/** The VM-exit exit qualification. */
185	uint64_t uExitQualification;
186
187	/** The VM-exit interruption-information field. */
188	uint32_t uExitIntrInfo;
189	/** The VM-exit instruction-length field. */
190	uint32_t cbInstr;
191	/** Whether the VM-entry failed or not. */
192	bool fVMEntryFailed;
193	/** Alignment. */
194	uint8_t abAlignment1[5];
195
196	/** The VM-entry interruption-information field. */
197	uint32_t uEntryIntrInfo;
198	/** The VM-entry exception error code field. */
199	uint32_t uEntryXcptErrorCode;
200	/** The VM-entry instruction length field. */
201	uint32_t cbEntryInstr;
202
203	/** IDT-vectoring information field. */
204	uint32_t uIdtVectoringInfo;
205	/** IDT-vectoring error code. */
206	uint32_t uIdtVectoringErrorCode;
207
208	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
209	uint32_t fVmcsFieldsRead;
210	/** Whether TSC-offsetting should be setup before VM-entry. */
211	bool fUpdateTscOffsettingAndPreemptTimer;
212	/** Whether the VM-exit was caused by a page-fault during delivery of a
213	* contributary exception or a page-fault. */
214	bool fVectoringPF;
215	} VMXTRANSIENT, *PVMXTRANSIENT;
216	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
217	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
218	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
219
220
221	/**
222	* MSR-bitmap read permissions.
223	*/
224	typedef enum VMXMSREXITREAD
225	{
226	/** Reading this MSR causes a VM-exit. */
227	VMXMSREXIT_INTERCEPT_READ = 0xb,
228	/** Reading this MSR does not cause a VM-exit. */
229	VMXMSREXIT_PASSTHRU_READ
230	} VMXMSREXITREAD;
231
232	/**
233	* MSR-bitmap write permissions.
234	*/
235	typedef enum VMXMSREXITWRITE
236	{
237	/** Writing to this MSR causes a VM-exit. */
238	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
239	/** Writing to this MSR does not cause a VM-exit. */
240	VMXMSREXIT_PASSTHRU_WRITE
241	} VMXMSREXITWRITE;
242
243
244	/*******************************************************************************
245	* Internal Functions *
246	*******************************************************************************/
247	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
248	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
249	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
250	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
251	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
252	#endif
253	#ifndef HMVMX_USE_FUNCTION_TABLE
254	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
255	#define HMVMX_EXIT_DECL static int
256	#else
257	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
258	#endif
259
260	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304
305	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312
313
314	/*******************************************************************************
315	* Global Variables *
316	*******************************************************************************/
317	#ifdef HMVMX_USE_FUNCTION_TABLE
318	/**
319	* VM-exit handler.
320	*
321	* @returns VBox status code.
322	* @param pVCpu Pointer to the VMCPU.
323	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
324	* out-of-sync. Make sure to update the required
325	* fields before using them.
326	* @param pVmxTransient Pointer to the VMX-transient structure.
327	*/
328	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
331
332	/**
333	* VMX_EXIT dispatch table.
334	*/
335	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
336	{
337	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
338	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
339	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
340	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
341	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
342	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
343	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
344	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
345	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
346	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
347	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
348	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
349	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
350	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
351	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
352	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
353	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
354	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
355	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
356	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
357	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
358	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
359	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
360	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
361	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
362	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
363	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
364	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
365	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
366	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
367	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
368	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
369	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
370	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
371	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
372	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
373	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
374	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
375	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
376	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
377	/* 40 UNDEFINED */ hmR0VmxExitPause,
378	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
379	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
380	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
381	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
382	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
383	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
384	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
385	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
386	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
387	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
388	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
389	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
390	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
391	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
392	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
393	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
394	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
395	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
396	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
397	};
398	#endif /* HMVMX_USE_FUNCTION_TABLE */
399
400	#ifdef VBOX_STRICT
401	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
402	{
403	/* 0 */ "(Not Used)",
404	/* 1 */ "VMCALL executed in VMX root operation.",
405	/* 2 */ "VMCLEAR with invalid physical address.",
406	/* 3 */ "VMCLEAR with VMXON pointer.",
407	/* 4 */ "VMLAUNCH with non-clear VMCS.",
408	/* 5 */ "VMRESUME with non-launched VMCS.",
409	/* 6 */ "VMRESUME after VMXOFF",
410	/* 7 */ "VM entry with invalid control fields.",
411	/* 8 */ "VM entry with invalid host state fields.",
412	/* 9 */ "VMPTRLD with invalid physical address.",
413	/* 10 */ "VMPTRLD with VMXON pointer.",
414	/* 11 */ "VMPTRLD with incorrect revision identifier.",
415	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
416	/* 13 */ "VMWRITE to read-only VMCS component.",
417	/* 14 */ "(Not Used)",
418	/* 15 */ "VMXON executed in VMX root operation.",
419	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
420	/* 17 */ "VM entry with non-launched executing VMCS.",
421	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
422	/* 19 */ "VMCALL with non-clear VMCS.",
423	/* 20 */ "VMCALL with invalid VM-exit control fields.",
424	/* 21 */ "(Not Used)",
425	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
426	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
427	/* 24 */ "VMCALL with invalid SMM-monitor features.",
428	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
429	/* 26 */ "VM entry with events blocked by MOV SS.",
430	/* 27 */ "(Not Used)",
431	/* 28 */ "Invalid operand to INVEPT/INVVPID."
432	};
433	#endif /* VBOX_STRICT */
434
435
436
437	/**
438	* Updates the VM's last error record. If there was a VMX instruction error,
439	* reads the error data from the VMCS and updates VCPU's last error record as
440	* well.
441	*
442	* @param pVM Pointer to the VM.
443	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
444	* VERR_VMX_UNABLE_TO_START_VM or
445	* VERR_VMX_INVALID_VMCS_FIELD).
446	* @param rc The error code.
447	*/
448	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
449	{
450	AssertPtr(pVM);
451	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
452	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
453	{
454	AssertPtrReturnVoid(pVCpu);
455	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
456	}
457	pVM->hm.s.lLastError = rc;
458	}
459
460
461	/**
462	* Reads the VM-entry interruption-information field from the VMCS into the VMX
463	* transient structure.
464	*
465	* @returns VBox status code.
466	* @param pVmxTransient Pointer to the VMX transient structure.
467	*
468	* @remarks No-long-jump zone!!!
469	*/
470	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
471	{
472	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
473	AssertRCReturn(rc, rc);
474	return VINF_SUCCESS;
475	}
476
477
478	/**
479	* Reads the VM-entry exception error code field from the VMCS into
480	* the VMX transient structure.
481	*
482	* @returns VBox status code.
483	* @param pVmxTransient Pointer to the VMX transient structure.
484	*
485	* @remarks No-long-jump zone!!!
486	*/
487	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
488	{
489	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
490	AssertRCReturn(rc, rc);
491	return VINF_SUCCESS;
492	}
493
494
495	/**
496	* Reads the VM-entry exception error code field from the VMCS into
497	* the VMX transient structure.
498	*
499	* @returns VBox status code.
500	* @param pVCpu Pointer to the VMCPU.
501	* @param pVmxTransient Pointer to the VMX transient structure.
502	*
503	* @remarks No-long-jump zone!!!
504	*/
505	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
506	{
507	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
508	AssertRCReturn(rc, rc);
509	return VINF_SUCCESS;
510	}
511
512
513	/**
514	* Reads the VM-exit interruption-information field from the VMCS into the VMX
515	* transient structure.
516	*
517	* @returns VBox status code.
518	* @param pVCpu Pointer to the VMCPU.
519	* @param pVmxTransient Pointer to the VMX transient structure.
520	*/
521	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
522	{
523	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
524	{
525	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
526	AssertRCReturn(rc, rc);
527	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
528	}
529	return VINF_SUCCESS;
530	}
531
532
533	/**
534	* Reads the VM-exit interruption error code from the VMCS into the VMX
535	* transient structure.
536	*
537	* @returns VBox status code.
538	* @param pVCpu Pointer to the VMCPU.
539	* @param pVmxTransient Pointer to the VMX transient structure.
540	*/
541	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
542	{
543	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
544	{
545	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
546	AssertRCReturn(rc, rc);
547	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
548	}
549	return VINF_SUCCESS;
550	}
551
552
553	/**
554	* Reads the VM-exit instruction length field from the VMCS into the VMX
555	* transient structure.
556	*
557	* @returns VBox status code.
558	* @param pVCpu Pointer to the VMCPU.
559	* @param pVmxTransient Pointer to the VMX transient structure.
560	*/
561	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
562	{
563	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
564	{
565	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
566	AssertRCReturn(rc, rc);
567	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
568	}
569	return VINF_SUCCESS;
570	}
571
572
573	/**
574	* Reads the exit qualification from the VMCS into the VMX transient structure.
575	*
576	* @returns VBox status code.
577	* @param pVCpu Pointer to the VMCPU.
578	* @param pVmxTransient Pointer to the VMX transient structure.
579	*/
580	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
581	{
582	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
583	{
584	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
585	AssertRCReturn(rc, rc);
586	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
587	}
588	return VINF_SUCCESS;
589	}
590
591
592	/**
593	* Reads the IDT-vectoring information field from the VMCS into the VMX
594	* transient structure.
595	*
596	* @returns VBox status code.
597	* @param pVmxTransient Pointer to the VMX transient structure.
598	*
599	* @remarks No-long-jump zone!!!
600	*/
601	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
602	{
603	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
604	{
605	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
606	AssertRCReturn(rc, rc);
607	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
608	}
609	return VINF_SUCCESS;
610	}
611
612
613	/**
614	* Reads the IDT-vectoring error code from the VMCS into the VMX
615	* transient structure.
616	*
617	* @returns VBox status code.
618	* @param pVmxTransient Pointer to the VMX transient structure.
619	*/
620	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
621	{
622	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
623	{
624	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
625	AssertRCReturn(rc, rc);
626	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
627	}
628	return VINF_SUCCESS;
629	}
630
631
632	/**
633	* Enters VMX root mode operation on the current CPU.
634	*
635	* @returns VBox status code.
636	* @param pVM Pointer to the VM (optional, can be NULL, after
637	* a resume).
638	* @param HCPhysCpuPage Physical address of the VMXON region.
639	* @param pvCpuPage Pointer to the VMXON region.
640	*/
641	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
642	{
643	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
644	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
645	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
646
647	if (pVM)
648	{
649	/* Write the VMCS revision dword to the VMXON region. */
650	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
651	}
652
653	/* Enable the VMX bit in CR4 if necessary. */
654	RTCCUINTREG uCr4 = ASMGetCR4();
655	if (!(uCr4 & X86_CR4_VMXE))
656	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
657
658	/* Enter VMX root mode. */
659	int rc = VMXEnable(HCPhysCpuPage);
660	if (RT_FAILURE(rc))
661	ASMSetCR4(uCr4);
662
663	return rc;
664	}
665
666
667	/**
668	* Exits VMX root mode operation on the current CPU.
669	*
670	* @returns VBox status code.
671	*/
672	static int hmR0VmxLeaveRootMode(void)
673	{
674	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
675
676	/* If we're for some reason not in VMX root mode, then don't leave it. */
677	if (ASMGetCR4() & X86_CR4_VMXE)
678	{
679	/* Exit VMX root mode and clear the VMX bit in CR4 */
680	VMXDisable();
681	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
682	return VINF_SUCCESS;
683	}
684
685	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
686	}
687
688
689	/**
690	* Allocates and maps one physically contiguous page. The allocated page is
691	* zero'd out. (Used by various VT-x structures).
692	*
693	* @returns IPRT status code.
694	* @param pMemObj Pointer to the ring-0 memory object.
695	* @param ppVirt Where to store the virtual address of the
696	* allocation.
697	* @param pPhys Where to store the physical address of the
698	* allocation.
699	*/
700	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
701	{
702	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
703	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
704	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
705
706	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
707	if (RT_FAILURE(rc))
708	return rc;
709	ppVirt = RTR0MemObjAddress(pMemObj);
710	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
711	ASMMemZero32(*ppVirt, PAGE_SIZE);
712	return VINF_SUCCESS;
713	}
714
715
716	/**
717	* Frees and unmaps an allocated physical page.
718	*
719	* @param pMemObj Pointer to the ring-0 memory object.
720	* @param ppVirt Where to re-initialize the virtual address of
721	* allocation as 0.
722	* @param pHCPhys Where to re-initialize the physical address of the
723	* allocation as 0.
724	*/
725	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
726	{
727	AssertPtr(pMemObj);
728	AssertPtr(ppVirt);
729	AssertPtr(pHCPhys);
730	if (*pMemObj != NIL_RTR0MEMOBJ)
731	{
732	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
733	AssertRC(rc);
734	*pMemObj = NIL_RTR0MEMOBJ;
735	*ppVirt = 0;
736	*pHCPhys = 0;
737	}
738	}
739
740
741	/**
742	* Worker function to free VT-x related structures.
743	*
744	* @returns IPRT status code.
745	* @param pVM Pointer to the VM.
746	*/
747	static void hmR0VmxStructsFree(PVM pVM)
748	{
749	for (VMCPUID i = 0; i < pVM->cCpus; i++)
750	{
751	PVMCPU pVCpu = &pVM->aCpus[i];
752	AssertPtr(pVCpu);
753
754	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
755	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
756	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
757	#endif
758
759	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
761
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
763	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
764	}
765
766	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
767	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
768	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
769	#endif
770	}
771
772
773	/**
774	* Worker function to allocate VT-x related VM structures.
775	*
776	* @returns IPRT status code.
777	* @param pVM Pointer to the VM.
778	*/
779	static int hmR0VmxStructsAlloc(PVM pVM)
780	{
781	/*
782	* Initialize members up-front so we can cleanup properly on allocation failure.
783	*/
784	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
785	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
786	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
787	pVM->hm.s.vmx.HCPhys##a_Name = 0;
788
789	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
790	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
791	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
792	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
793
794	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
795	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
796	#endif
797	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
798
799	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
800	for (VMCPUID i = 0; i < pVM->cCpus; i++)
801	{
802	PVMCPU pVCpu = &pVM->aCpus[i];
803	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
804	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
805	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
806	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
809	#endif
810	}
811	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
812	#undef VMXLOCAL_INIT_VM_MEMOBJ
813
814	/*
815	* Allocate all the VT-x structures.
816	*/
817	int rc = VINF_SUCCESS;
818	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
819	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
820	if (RT_FAILURE(rc))
821	goto cleanup;
822	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
823	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
824	#endif
825
826	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
827	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
828	{
829	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
830	&pVM->hm.s.vmx.HCPhysApicAccess);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833	}
834
835	/*
836	* Initialize per-VCPU VT-x structures.
837	*/
838	for (VMCPUID i = 0; i < pVM->cCpus; i++)
839	{
840	PVMCPU pVCpu = &pVM->aCpus[i];
841	AssertPtr(pVCpu);
842
843	/* Allocate the VM control structure (VMCS). */
844	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
845	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
846	if (RT_FAILURE(rc))
847	goto cleanup;
848
849	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
850	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
851	{
852	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
853	&pVCpu->hm.s.vmx.HCPhysVirtApic);
854	if (RT_FAILURE(rc))
855	goto cleanup;
856	}
857
858	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
859	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
860	{
861	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
862	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
863	if (RT_FAILURE(rc))
864	goto cleanup;
865	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
866	}
867
868	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
869	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
870	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
871	if (RT_FAILURE(rc))
872	goto cleanup;
873
874	/* Allocate the VM-exit MSR-load page for the host MSRs. */
875	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
876	if (RT_FAILURE(rc))
877	goto cleanup;
878	#endif
879	}
880
881	return VINF_SUCCESS;
882
883	cleanup:
884	hmR0VmxStructsFree(pVM);
885	return rc;
886	}
887
888
889	/**
890	* Does global VT-x initialization (called during module initialization).
891	*
892	* @returns VBox status code.
893	*/
894	VMMR0DECL(int) VMXR0GlobalInit(void)
895	{
896	#ifdef HMVMX_USE_FUNCTION_TABLE
897	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
898	# ifdef VBOX_STRICT
899	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
900	Assert(g_apfnVMExitHandlers[i]);
901	# endif
902	#endif
903	return VINF_SUCCESS;
904	}
905
906
907	/**
908	* Does global VT-x termination (called during module termination).
909	*/
910	VMMR0DECL(void) VMXR0GlobalTerm()
911	{
912	/* Nothing to do currently. */
913	}
914
915
916	/**
917	* Sets up and activates VT-x on the current CPU.
918	*
919	* @returns VBox status code.
920	* @param pCpu Pointer to the global CPU info struct.
921	* @param pVM Pointer to the VM (can be NULL after a host resume
922	* operation).
923	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
924	* fEnabledByHost is true).
925	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
926	* @a fEnabledByHost is true).
927	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
928	* enable VT-x/AMD-V on the host.
929	*/
930	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
931	{
932	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
933	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
934
935	if (!fEnabledByHost)
936	{
937	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
938	if (RT_FAILURE(rc))
939	return rc;
940	}
941
942	/*
943	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
944	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
945	* each time while reusing a VPID after hitting the MaxASID limit once.
946	*/
947	if ( pVM
948	&& pVM->hm.s.vmx.fVpid
949	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
950	{
951	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
952	pCpu->fFlushAsidBeforeUse = false;
953	}
954	else
955	pCpu->fFlushAsidBeforeUse = true;
956
957	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
958	++pCpu->cTlbFlushes;
959
960	return VINF_SUCCESS;
961	}
962
963
964	/**
965	* Deactivates VT-x on the current CPU.
966	*
967	* @returns VBox status code.
968	* @param pCpu Pointer to the global CPU info struct.
969	* @param pvCpuPage Pointer to the VMXON region.
970	* @param HCPhysCpuPage Physical address of the VMXON region.
971	*/
972	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
973	{
974	NOREF(pCpu);
975	NOREF(pvCpuPage);
976	NOREF(HCPhysCpuPage);
977
978	return hmR0VmxLeaveRootMode();
979	}
980
981
982	/**
983	* Sets the permission bits for the specified MSR in the MSR bitmap.
984	*
985	* @param pVCpu Pointer to the VMCPU.
986	* @param uMSR The MSR value.
987	* @param enmRead Whether reading this MSR causes a VM-exit.
988	* @param enmWrite Whether writing this MSR causes a VM-exit.
989	*/
990	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
991	{
992	int32_t iBit;
993	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
994
995	/*
996	* Layout:
997	* 0x000 - 0x3ff - Low MSR read bits
998	* 0x400 - 0x7ff - High MSR read bits
999	* 0x800 - 0xbff - Low MSR write bits
1000	* 0xc00 - 0xfff - High MSR write bits
1001	*/
1002	if (uMsr <= 0x00001FFF)
1003	iBit = uMsr;
1004	else if ( uMsr >= 0xC0000000
1005	&& uMsr <= 0xC0001FFF)
1006	{
1007	iBit = (uMsr - 0xC0000000);
1008	pbMsrBitmap += 0x400;
1009	}
1010	else
1011	{
1012	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1013	return;
1014	}
1015
1016	Assert(iBit <= 0x1fff);
1017	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1018	ASMBitSet(pbMsrBitmap, iBit);
1019	else
1020	ASMBitClear(pbMsrBitmap, iBit);
1021
1022	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1023	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1024	else
1025	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1026	}
1027
1028
1029	/**
1030	* Flushes the TLB using EPT.
1031	*
1032	* @returns VBox status code.
1033	* @param pVM Pointer to the VM.
1034	* @param pVCpu Pointer to the VMCPU.
1035	* @param enmFlush Type of flush.
1036	*/
1037	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1038	{
1039	AssertPtr(pVM);
1040	Assert(pVM->hm.s.fNestedPaging);
1041
1042	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1043
1044	uint64_t descriptor[2];
1045	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1046	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1047
1048	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1049	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1050	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1051	}
1052
1053
1054	/**
1055	* Flushes the TLB using VPID.
1056	*
1057	* @returns VBox status code.
1058	* @param pVM Pointer to the VM.
1059	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1060	* enmFlush).
1061	* @param enmFlush Type of flush.
1062	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1063	* on @a enmFlush).
1064	*/
1065	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1066	{
1067	AssertPtr(pVM);
1068	Assert(pVM->hm.s.vmx.fVpid);
1069
1070	uint64_t descriptor[2];
1071	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1072	{
1073	descriptor[0] = 0;
1074	descriptor[1] = 0;
1075	}
1076	else
1077	{
1078	AssertPtr(pVCpu);
1079	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1080	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1081	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1082	descriptor[1] = GCPtr;
1083	}
1084
1085	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1086	AssertMsg(rc == VINF_SUCCESS,
1087	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1088	if ( RT_SUCCESS(rc)
1089	&& pVCpu)
1090	{
1091	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1092	}
1093	}
1094
1095
1096	/**
1097	* Invalidates a guest page by guest virtual address. Only relevant for
1098	* EPT/VPID, otherwise there is nothing really to invalidate.
1099	*
1100	* @returns VBox status code.
1101	* @param pVM Pointer to the VM.
1102	* @param pVCpu Pointer to the VMCPU.
1103	* @param GCVirt Guest virtual address of the page to invalidate.
1104	*/
1105	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1106	{
1107	AssertPtr(pVM);
1108	AssertPtr(pVCpu);
1109	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1110
1111	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1112	if (!fFlushPending)
1113	{
1114	/*
1115	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1116	* See @bugref{6043} and @bugref{6177}.
1117	*
1118	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1119	* function maybe called in a loop with individual addresses.
1120	*/
1121	if (pVM->hm.s.vmx.fVpid)
1122	{
1123	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1124	{
1125	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1126	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1127	}
1128	else
1129	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1130	}
1131	else if (pVM->hm.s.fNestedPaging)
1132	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1133	}
1134
1135	return VINF_SUCCESS;
1136	}
1137
1138
1139	/**
1140	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1141	* otherwise there is nothing really to invalidate.
1142	*
1143	* @returns VBox status code.
1144	* @param pVM Pointer to the VM.
1145	* @param pVCpu Pointer to the VMCPU.
1146	* @param GCPhys Guest physical address of the page to invalidate.
1147	*/
1148	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1149	{
1150	LogFlowFunc(("%RGp\n", GCPhys));
1151
1152	/*
1153	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1154	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1155	* This function might be called in a loop.
1156	*/
1157	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1158	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1159	return VINF_SUCCESS;
1160	}
1161
1162
1163	/**
1164	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1165	* case where neither EPT nor VPID is supported by the CPU.
1166	*
1167	* @param pVM Pointer to the VM.
1168	* @param pVCpu Pointer to the VMCPU.
1169	*
1170	* @remarks Called with interrupts disabled.
1171	*/
1172	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1173	{
1174	NOREF(pVM);
1175	AssertPtr(pVCpu);
1176	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1177	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1178
1179	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1180	AssertPtr(pCpu);
1181
1182	pVCpu->hm.s.TlbShootdown.cPages = 0;
1183	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1184	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1185	pVCpu->hm.s.fForceTLBFlush = false;
1186	return;
1187	}
1188
1189
1190	/**
1191	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1192	*
1193	* @param pVM Pointer to the VM.
1194	* @param pVCpu Pointer to the VMCPU.
1195	* @remarks All references to "ASID" in this function pertains to "VPID" in
1196	* Intel's nomenclature. The reason is, to avoid confusion in compare
1197	* statements since the host-CPU copies are named "ASID".
1198	*
1199	* @remarks Called with interrupts disabled.
1200	*/
1201	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1202	{
1203	AssertPtr(pVM);
1204	AssertPtr(pVCpu);
1205	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1206	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1207	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1208
1209	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1210	AssertPtr(pCpu);
1211
1212	/*
1213	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1214	* This can happen both for start & resume due to long jumps back to ring-3.
1215	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1216	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1217	*/
1218	bool fNewASID = false;
1219	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1220	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1221	{
1222	pVCpu->hm.s.fForceTLBFlush = true;
1223	fNewASID = true;
1224	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1225	}
1226
1227	/*
1228	* Check for explicit TLB shootdowns.
1229	*/
1230	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1231	{
1232	pVCpu->hm.s.fForceTLBFlush = true;
1233	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1234	}
1235
1236	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1237	if (pVCpu->hm.s.fForceTLBFlush)
1238	{
1239	if (fNewASID)
1240	{
1241	++pCpu->uCurrentAsid;
1242	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1243	{
1244	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1245	pCpu->cTlbFlushes++;
1246	pCpu->fFlushAsidBeforeUse = true;
1247	}
1248
1249	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1250	if (pCpu->fFlushAsidBeforeUse)
1251	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1252	}
1253	else
1254	{
1255	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1256	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1257	else
1258	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1259	}
1260
1261	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1262	pVCpu->hm.s.fForceTLBFlush = false;
1263	}
1264	else
1265	{
1266	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1267	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1268	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1269	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1270
1271	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1272	* not be executed. See hmQueueInvlPage() where it is commented
1273	* out. Support individual entry flushing someday. */
1274	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1275	{
1276	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1277
1278	/*
1279	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1280	* as supported by the CPU.
1281	*/
1282	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1283	{
1284	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1285	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1286	}
1287	else
1288	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1289	}
1290	else
1291	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1292	}
1293	pVCpu->hm.s.TlbShootdown.cPages = 0;
1294	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1295
1296	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1297	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1298	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1299	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1300	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1301	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1302
1303	/* Update VMCS with the VPID. */
1304	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1305	AssertRC(rc);
1306	}
1307
1308
1309	/**
1310	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1311	*
1312	* @returns VBox status code.
1313	* @param pVM Pointer to the VM.
1314	* @param pVCpu Pointer to the VMCPU.
1315	*
1316	* @remarks Called with interrupts disabled.
1317	*/
1318	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1319	{
1320	AssertPtr(pVM);
1321	AssertPtr(pVCpu);
1322	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1323	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1324
1325	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1326	AssertPtr(pCpu);
1327
1328	/*
1329	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1330	* This can happen both for start & resume due to long jumps back to ring-3.
1331	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1332	*/
1333	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1334	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1335	{
1336	pVCpu->hm.s.fForceTLBFlush = true;
1337	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1338	}
1339
1340	/* Check for explicit TLB shootdown flushes. */
1341	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1342	{
1343	pVCpu->hm.s.fForceTLBFlush = true;
1344	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1345	}
1346
1347	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1348	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1349
1350	if (pVCpu->hm.s.fForceTLBFlush)
1351	{
1352	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1353	pVCpu->hm.s.fForceTLBFlush = false;
1354	}
1355	else
1356	{
1357	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1358	* not be executed. See hmQueueInvlPage() where it is commented
1359	* out. Support individual entry flushing someday. */
1360	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1361	{
1362	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1363	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1364	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1365	}
1366	else
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1368	}
1369
1370	pVCpu->hm.s.TlbShootdown.cPages = 0;
1371	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1372	}
1373
1374
1375	/**
1376	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1377	*
1378	* @returns VBox status code.
1379	* @param pVM Pointer to the VM.
1380	* @param pVCpu Pointer to the VMCPU.
1381	*
1382	* @remarks Called with interrupts disabled.
1383	*/
1384	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1385	{
1386	AssertPtr(pVM);
1387	AssertPtr(pVCpu);
1388	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1389	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1390
1391	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1392
1393	/*
1394	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1395	* This can happen both for start & resume due to long jumps back to ring-3.
1396	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1397	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1398	*/
1399	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1400	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1401	{
1402	pVCpu->hm.s.fForceTLBFlush = true;
1403	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1404	}
1405
1406	/* Check for explicit TLB shootdown flushes. */
1407	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1408	{
1409	/*
1410	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1411	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1412	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1413	*/
1414	pVCpu->hm.s.fForceTLBFlush = true;
1415	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1416	}
1417
1418	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1419	if (pVCpu->hm.s.fForceTLBFlush)
1420	{
1421	++pCpu->uCurrentAsid;
1422	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1423	{
1424	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1425	pCpu->fFlushAsidBeforeUse = true;
1426	pCpu->cTlbFlushes++;
1427	}
1428
1429	pVCpu->hm.s.fForceTLBFlush = false;
1430	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1431	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1432	if (pCpu->fFlushAsidBeforeUse)
1433	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1434	}
1435	else
1436	{
1437	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1438	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1439	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1440	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1441
1442	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1443	* not be executed. See hmQueueInvlPage() where it is commented
1444	* out. Support individual entry flushing someday. */
1445	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1446	{
1447	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1448	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1449	{
1450	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1451	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1452	}
1453	else
1454	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1455	}
1456	else
1457	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1458	}
1459
1460	pVCpu->hm.s.TlbShootdown.cPages = 0;
1461	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1462
1463	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1464	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1465	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1466	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1467	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1468	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1469
1470	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1471	AssertRC(rc);
1472	}
1473
1474
1475	/**
1476	* Flushes the guest TLB entry based on CPU capabilities.
1477	*
1478	* @param pVCpu Pointer to the VMCPU.
1479	*/
1480	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1481	{
1482	PVM pVM = pVCpu->CTX_SUFF(pVM);
1483	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1484	{
1485	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1486	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1487	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1488	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1489	default:
1490	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1491	break;
1492	}
1493	}
1494
1495
1496	/**
1497	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1498	* TLB entries from the host TLB before VM-entry.
1499	*
1500	* @returns VBox status code.
1501	* @param pVM Pointer to the VM.
1502	*/
1503	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1504	{
1505	/*
1506	* Determine optimal flush type for nested paging.
1507	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1508	* guest execution (see hmR3InitFinalizeR0()).
1509	*/
1510	if (pVM->hm.s.fNestedPaging)
1511	{
1512	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1513	{
1514	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1515	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1516	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1517	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1518	else
1519	{
1520	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1521	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1522	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1523	}
1524
1525	/* Make sure the write-back cacheable memory type for EPT is supported. */
1526	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1527	{
1528	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1529	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1531	}
1532	}
1533	else
1534	{
1535	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1536	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1537	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1538	}
1539	}
1540
1541	/*
1542	* Determine optimal flush type for VPID.
1543	*/
1544	if (pVM->hm.s.vmx.fVpid)
1545	{
1546	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1547	{
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1549	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1550	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1552	else
1553	{
1554	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1555	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1556	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1557	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1558	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1559	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1560	pVM->hm.s.vmx.fVpid = false;
1561	}
1562	}
1563	else
1564	{
1565	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1566	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1567	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1568	pVM->hm.s.vmx.fVpid = false;
1569	}
1570	}
1571
1572	/*
1573	* Setup the handler for flushing tagged-TLBs.
1574	*/
1575	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1576	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1577	else if (pVM->hm.s.fNestedPaging)
1578	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1579	else if (pVM->hm.s.vmx.fVpid)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1581	else
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1583	return VINF_SUCCESS;
1584	}
1585
1586
1587	/**
1588	* Sets up pin-based VM-execution controls in the VMCS.
1589	*
1590	* @returns VBox status code.
1591	* @param pVM Pointer to the VM.
1592	* @param pVCpu Pointer to the VMCPU.
1593	*/
1594	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1595	{
1596	AssertPtr(pVM);
1597	AssertPtr(pVCpu);
1598
1599	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1600	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1601
1602	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1603	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1604	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1605
1606	/* Enable the VMX preemption timer. */
1607	if (pVM->hm.s.vmx.fUsePreemptTimer)
1608	{
1609	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1610	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1611	}
1612
1613	if ((val & zap) != val)
1614	{
1615	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1616	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1617	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1618	}
1619
1620	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1621	AssertRCReturn(rc, rc);
1622
1623	/* Update VCPU with the currently set pin-based VM-execution controls. */
1624	pVCpu->hm.s.vmx.u32PinCtls = val;
1625	return rc;
1626	}
1627
1628
1629	/**
1630	* Sets up processor-based VM-execution controls in the VMCS.
1631	*
1632	* @returns VBox status code.
1633	* @param pVM Pointer to the VM.
1634	* @param pVMCPU Pointer to the VMCPU.
1635	*/
1636	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1637	{
1638	AssertPtr(pVM);
1639	AssertPtr(pVCpu);
1640
1641	int rc = VERR_INTERNAL_ERROR_5;
1642	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1643	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1644
1645	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1646	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1647	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1648	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1649	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1652
1653	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1654	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1655	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1656	{
1657	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1658	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1659	}
1660
1661	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1662	if (!pVM->hm.s.fNestedPaging)
1663	{
1664	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1665	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1666	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1667	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1668	}
1669
1670	/* Use TPR shadowing if supported by the CPU. */
1671	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1672	{
1673	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1674	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1675	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1676	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1677	AssertRCReturn(rc, rc);
1678
1679	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1680	/* CR8 writes causes a VM-exit based on TPR threshold. */
1681	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1682	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1683	}
1684	else
1685	{
1686	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1687	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1688	}
1689
1690	/* Use MSR-bitmaps if supported by the CPU. */
1691	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1692	{
1693	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1694
1695	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1696	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1697	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1698	AssertRCReturn(rc, rc);
1699
1700	/*
1701	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1702	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1703	*/
1704	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1705	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1706	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1707	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	}
1714
1715	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1716	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1717	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1718
1719	if ((val & zap) != val)
1720	{
1721	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1722	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1723	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1724	}
1725
1726	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1727	AssertRCReturn(rc, rc);
1728
1729	/* Update VCPU with the currently set processor-based VM-execution controls. */
1730	pVCpu->hm.s.vmx.u32ProcCtls = val;
1731
1732	/*
1733	* Secondary processor-based VM-execution controls.
1734	*/
1735	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1736	{
1737	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1738	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1739
1740	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1741	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1742
1743	if (pVM->hm.s.fNestedPaging)
1744	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1745	else
1746	{
1747	/*
1748	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1749	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1750	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1751	*/
1752	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1753	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1754	}
1755
1756	if (pVM->hm.s.vmx.fVpid)
1757	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1758
1759	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1760	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1761
1762	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1763	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1764	* done dynamically. */
1765	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1766	{
1767	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1768	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1769	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1770	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1771	AssertRCReturn(rc, rc);
1772	}
1773
1774	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1775	{
1776	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1778	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1779	}
1780
1781	if ((val & zap) != val)
1782	{
1783	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1784	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1785	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1786	}
1787
1788	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1789	AssertRCReturn(rc, rc);
1790
1791	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1792	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1793	}
1794
1795	return VINF_SUCCESS;
1796	}
1797
1798
1799	/**
1800	* Sets up miscellaneous (everything other than Pin & Processor-based
1801	* VM-execution) control fields in the VMCS.
1802	*
1803	* @returns VBox status code.
1804	* @param pVM Pointer to the VM.
1805	* @param pVCpu Pointer to the VMCPU.
1806	*/
1807	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1808	{
1809	AssertPtr(pVM);
1810	AssertPtr(pVCpu);
1811
1812	/** @todo Shouldn't we able to avoid initializing with 0? */
1813	int rc = VERR_GENERAL_FAILURE;
1814
1815	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1816	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1817	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1818
1819	/*
1820	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1821	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1822	* We thus use the exception bitmap to control it rather than use both.
1823	*/
1824	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1825	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1826
1827	/** @todo Explore possibility of using IO-bitmaps. */
1828	/* All IO & IOIO instructions cause VM-exits. */
1829	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1830	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1831
1832	/* Initialize the MSR-bitmap area. */
1833	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1834	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1835	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1836
1837	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1838	/* Setup MSR autoloading/storing. */
1839	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1840	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1841	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	AssertRCReturn(rc, rc);
1843	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	AssertRCReturn(rc, rc);
1845
1846	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1847	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1848	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1849	AssertRCReturn(rc, rc);
1850	#endif
1851
1852	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1853	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1854	AssertRCReturn(rc, rc);
1855
1856	/* Setup debug controls */
1857	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1858	AssertRCReturn(rc, rc);
1859	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1860	AssertRCReturn(rc, rc);
1861	return rc;
1862	}
1863
1864
1865	/**
1866	* Sets up the initial exception bitmap in the VMCS based on static conditions
1867	* (i.e. conditions that cannot ever change at runtime).
1868	*
1869	* @returns VBox status code.
1870	* @param pVM Pointer to the VM.
1871	* @param pVCpu Pointer to the VMCPU.
1872	*/
1873	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1874	{
1875	AssertPtr(pVM);
1876	AssertPtr(pVCpu);
1877
1878	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1879
1880	uint32_t u32XcptBitmap = 0;
1881
1882	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1883	if (!pVM->hm.s.fNestedPaging)
1884	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1885
1886	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1887	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1888	AssertRCReturn(rc, rc);
1889	return rc;
1890	}
1891
1892
1893	/**
1894	* Sets up the initial guest-state mask. The guest-state mask is consulted
1895	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1896	* for the nested virtualization case (as it would cause a VM-exit).
1897	*
1898	* @param pVCpu Pointer to the VMCPU.
1899	*/
1900	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1901	{
1902	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1903	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1904	return VINF_SUCCESS;
1905	}
1906
1907
1908	/**
1909	* Does per-VM VT-x initialization.
1910	*
1911	* @returns VBox status code.
1912	* @param pVM Pointer to the VM.
1913	*/
1914	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1915	{
1916	LogFlowFunc(("pVM=%p\n", pVM));
1917
1918	int rc = hmR0VmxStructsAlloc(pVM);
1919	if (RT_FAILURE(rc))
1920	{
1921	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1922	return rc;
1923	}
1924
1925	return VINF_SUCCESS;
1926	}
1927
1928
1929	/**
1930	* Does per-VM VT-x termination.
1931	*
1932	* @returns VBox status code.
1933	* @param pVM Pointer to the VM.
1934	*/
1935	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1936	{
1937	LogFlowFunc(("pVM=%p\n", pVM));
1938
1939	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1940	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1941	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1942	#endif
1943	hmR0VmxStructsFree(pVM);
1944	return VINF_SUCCESS;
1945	}
1946
1947
1948	/**
1949	* Sets up the VM for execution under VT-x.
1950	* This function is only called once per-VM during initalization.
1951	*
1952	* @returns VBox status code.
1953	* @param pVM Pointer to the VM.
1954	*/
1955	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1956	{
1957	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1958	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1959
1960	LogFlowFunc(("pVM=%p\n", pVM));
1961
1962	/*
1963	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1964	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1965	*/
1966	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1967	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1968	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1969	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1970	{
1971	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1972	return VERR_INTERNAL_ERROR;
1973	}
1974
1975	/* Initialize these always, see hmR3InitFinalizeR0().*/
1976	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1977	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1978
1979	/* Setup the tagged-TLB flush handlers. */
1980	int rc = hmR0VmxSetupTaggedTlb(pVM);
1981	if (RT_FAILURE(rc))
1982	{
1983	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1984	return rc;
1985	}
1986
1987	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1988	{
1989	PVMCPU pVCpu = &pVM->aCpus[i];
1990	AssertPtr(pVCpu);
1991	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1992
1993	/* Set revision dword at the beginning of the VMCS structure. */
1994	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1995
1996	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1997	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1998	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1999	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2000
2001	/* Load this VMCS as the current VMCS. */
2002	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2003	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2004	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2005
2006	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2007	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2008	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2009
2010	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2011	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2012	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2013
2014	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2015	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2016	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2017
2018	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2019	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2020	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2021
2022	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2023	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2024	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2025
2026	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2027	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2028	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2029	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2030	#endif
2031
2032	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2033	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2034	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2035	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2036
2037	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2038	}
2039
2040	return VINF_SUCCESS;
2041	}
2042
2043
2044	/**
2045	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2046	* the VMCS.
2047	*
2048	* @returns VBox status code.
2049	* @param pVM Pointer to the VM.
2050	* @param pVCpu Pointer to the VMCPU.
2051	*/
2052	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2053	{
2054	RTCCUINTREG uReg = ASMGetCR0();
2055	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2056	AssertRCReturn(rc, rc);
2057
2058	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2059	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2060	if (HMVMX_IS_64BIT_HOST_MODE())
2061	{
2062	uint64_t uRegCR3 = HMR0Get64bitCR3();
2063	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2064	}
2065	else
2066	#endif
2067	{
2068	uReg = ASMGetCR3();
2069	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2070	}
2071	AssertRCReturn(rc, rc);
2072
2073	uReg = ASMGetCR4();
2074	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2075	AssertRCReturn(rc, rc);
2076	return rc;
2077	}
2078
2079
2080	/**
2081	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2082	* the host-state area in the VMCS.
2083	*
2084	* @returns VBox status code.
2085	* @param pVM Pointer to the VM.
2086	* @param pVCpu Pointer to the VMCPU.
2087	*/
2088	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2089	{
2090	int rc = VERR_INTERNAL_ERROR_5;
2091	RTSEL uSelCS = 0;
2092	RTSEL uSelSS = 0;
2093	RTSEL uSelDS = 0;
2094	RTSEL uSelES = 0;
2095	RTSEL uSelFS = 0;
2096	RTSEL uSelGS = 0;
2097	RTSEL uSelTR = 0;
2098
2099	/*
2100	* Host Selector registers.
2101	*/
2102	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2103	if (HMVMX_IS_64BIT_HOST_MODE())
2104	{
2105	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2106	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2107	}
2108	else
2109	{
2110	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2111	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2112	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2113	}
2114	#else
2115	uSelCS = ASMGetCS();
2116	uSelSS = ASMGetSS();
2117	#endif
2118
2119	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2120	uSelTR = ASMGetTR();
2121
2122	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2123	/** @todo Verify if we have any platform that actually run with DS or ES with
2124	* RPL != 0 in kernel space. */
2125	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2126	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2127	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2128	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2129	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2130	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2131	Assert(uSelCS != 0);
2132	Assert(uSelTR != 0);
2133
2134	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2135	#if 0
2136	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2137	Assert(uSelSS != 0);
2138	#endif
2139
2140	/* Write these host selector fields into the host-state area in the VMCS. */
2141	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2142	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2143	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2144	#if 0
2145	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2146	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2147	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2148	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2149	#endif
2150	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2151
2152	/*
2153	* Host GDTR and IDTR.
2154	*/
2155	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2156	* be safe to -not- save and restore GDTR and IDTR in the assembly
2157	* code and just do it here and don't care if the limits are zapped on
2158	* VM-exit. */
2159	RTGDTR Gdtr;
2160	RT_ZERO(Gdtr);
2161	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2162	if (HMVMX_IS_64BIT_HOST_MODE())
2163	{
2164	X86XDTR64 Gdtr64;
2165	X86XDTR64 Idtr64;
2166	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2167	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2168	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2169
2170	Gdtr.cbGdt = Gdtr64.cb;
2171	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2172	}
2173	else
2174	#endif
2175	{
2176	RTIDTR Idtr;
2177	ASMGetGDTR(&Gdtr);
2178	ASMGetIDTR(&Idtr);
2179	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2180	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2181	}
2182
2183	/*
2184	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2185	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2186	*/
2187	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2188	{
2189	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2190	return VERR_VMX_INVALID_HOST_STATE;
2191	}
2192
2193	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2194	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2195	if (HMVMX_IS_64BIT_HOST_MODE())
2196	{
2197	/* We need the 64-bit TR base for hybrid darwin. */
2198	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2199	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2200	}
2201	else
2202	#endif
2203	{
2204	uintptr_t uTRBase;
2205	#if HC_ARCH_BITS == 64
2206	uTRBase = X86DESC64_BASE(pDesc);
2207	#else
2208	uTRBase = X86DESC_BASE(pDesc);
2209	#endif
2210	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2211	}
2212	AssertRCReturn(rc, rc);
2213
2214	/*
2215	* Host FS base and GS base.
2216	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2217	* would take care of the bases. In 64-bit, the MSRs come into play.
2218	*/
2219	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2220	if (HMVMX_IS_64BIT_HOST_MODE())
2221	{
2222	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2223	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2224	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2225	AssertRCReturn(rc, rc);
2226	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2227	AssertRCReturn(rc, rc);
2228	}
2229	#endif
2230	return rc;
2231	}
2232
2233
2234	/**
2235	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2236	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2237	* the host after every successful VM exit.
2238	*
2239	* @returns VBox status code.
2240	* @param pVM Pointer to the VM.
2241	* @param pVCpu Pointer to the VMCPU.
2242	*/
2243	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2244	{
2245	AssertPtr(pVCpu);
2246	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2247
2248	int rc = VINF_SUCCESS;
2249	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2250	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2251	uint32_t cHostMsrs = 0;
2252	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2253
2254	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2255	{
2256	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2257	pHostMsr->u32Reserved = 0;
2258	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2259	if (CPUMIsGuestInLongMode(pVCpu))
2260	{
2261	/* Must match the EFER value in our 64 bits switcher. */
2262	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2263	}
2264	else
2265	# endif
2266	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2267	pHostMsr++; cHostMsrs++;
2268	}
2269
2270	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2271	if (HMVMX_IS_64BIT_HOST_MODE())
2272	{
2273	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2274	pHostMsr->u32Reserved = 0;
2275	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2276	pHostMsr++; cHostMsrs++;
2277	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2278	pHostMsr->u32Reserved = 0;
2279	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2280	pHostMsr++; cHostMsrs++;
2281	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2282	pHostMsr->u32Reserved = 0;
2283	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2284	pHostMsr++; cHostMsrs++;
2285	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2286	pHostMsr->u32Reserved = 0;
2287	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2288	pHostMsr++; cHostMsrs++;
2289	}
2290	# endif
2291
2292	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2293	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2294	{
2295	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2297	}
2298
2299	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2300	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2301
2302	/*
2303	* Host Sysenter MSRs.
2304	*/
2305	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2306	AssertRCReturn(rc, rc);
2307	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2308	if (HMVMX_IS_64BIT_HOST_MODE())
2309	{
2310	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2311	AssertRCReturn(rc, rc);
2312	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2313	}
2314	else
2315	{
2316	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2317	AssertRCReturn(rc, rc);
2318	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2319	}
2320	# elif HC_ARCH_BITS == 32
2321	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2322	AssertRCReturn(rc, rc);
2323	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2324	# else
2325	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2326	AssertRCReturn(rc, rc);
2327	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2328	# endif
2329	AssertRCReturn(rc, rc);
2330
2331	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2332	* hmR0VmxSetupExitCtls() !! */
2333	return rc;
2334	}
2335
2336
2337	/**
2338	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2339	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2340	* controls".
2341	*
2342	* @returns VBox status code.
2343	* @param pVCpu Pointer to the VMCPU.
2344	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2345	* out-of-sync. Make sure to update the required fields
2346	* before using them.
2347	*
2348	* @remarks No-long-jump zone!!!
2349	*/
2350	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2351	{
2352	int rc = VINF_SUCCESS;
2353	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2354	{
2355	PVM pVM = pVCpu->CTX_SUFF(pVM);
2356	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2357	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2358
2359	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2360	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2361
2362	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2363	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2364	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2365	else
2366	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2367
2368	/*
2369	* The following should not be set (since we're not in SMM mode):
2370	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2371	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2372	*/
2373
2374	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2375	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2376	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2377
2378	if ((val & zap) != val)
2379	{
2380	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2381	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2382	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2383	}
2384
2385	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2386	AssertRCReturn(rc, rc);
2387
2388	/* Update VCPU with the currently set VM-exit controls. */
2389	pVCpu->hm.s.vmx.u32EntryCtls = val;
2390	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2391	}
2392	return rc;
2393	}
2394
2395
2396	/**
2397	* Sets up the VM-exit controls in the VMCS.
2398	*
2399	* @returns VBox status code.
2400	* @param pVM Pointer to the VM.
2401	* @param pVCpu Pointer to the VMCPU.
2402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2403	* out-of-sync. Make sure to update the required fields
2404	* before using them.
2405	*
2406	* @remarks requires EFER.
2407	*/
2408	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2409	{
2410	int rc = VINF_SUCCESS;
2411	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2412	{
2413	PVM pVM = pVCpu->CTX_SUFF(pVM);
2414	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2415	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2416
2417	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2418	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2419
2420	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2421	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2422	if (HMVMX_IS_64BIT_HOST_MODE())
2423	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2424	else
2425	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2426	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2427	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2428	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2429	else
2430	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2431	#endif
2432
2433	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2434	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2435
2436	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2437	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2438	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2439	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2440	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2441
2442	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2443	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2444
2445	if ((val & zap) != val)
2446	{
2447	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2448	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2449	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2450	}
2451
2452	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2453	AssertRCReturn(rc, rc);
2454
2455	/* Update VCPU with the currently set VM-exit controls. */
2456	pVCpu->hm.s.vmx.u32ExitCtls = val;
2457	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2458	}
2459	return rc;
2460	}
2461
2462
2463	/**
2464	* Loads the guest APIC and related state.
2465	*
2466	* @returns VBox status code.
2467	* @param pVM Pointer to the VM.
2468	* @param pVCpu Pointer to the VMCPU.
2469	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2470	* out-of-sync. Make sure to update the required fields
2471	* before using them.
2472	*/
2473	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2474	{
2475	int rc = VINF_SUCCESS;
2476	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2477	{
2478	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2479	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2480	{
2481	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2482
2483	bool fPendingIntr = false;
2484	uint8_t u8Tpr = 0;
2485	uint8_t u8PendingIntr = 0;
2486	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2487	AssertRCReturn(rc, rc);
2488
2489	/*
2490	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2491	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2492	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2493	* the interrupt when we VM-exit for other reasons.
2494	*/
2495	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2496	uint32_t u32TprThreshold = 0;
2497	if (fPendingIntr)
2498	{
2499	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2500	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2501	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2502	if (u8PendingPriority <= u8TprPriority)
2503	u32TprThreshold = u8PendingPriority;
2504	else
2505	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2506	}
2507	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2508
2509	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2510	AssertRCReturn(rc, rc);
2511
2512	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2513	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2514	{
2515	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2516	pMixedCtx->msrLSTAR = u8Tpr;
2517	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2518	{
2519	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2520	if (fPendingIntr)
2521	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2522	else
2523	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524	}
2525	}
2526	}
2527
2528	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2529	}
2530	return rc;
2531	}
2532
2533
2534	/**
2535	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2536	*
2537	* @returns
2538	* @param pVCpu Pointer to the VMCPU.
2539	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2540	* out-of-sync. Make sure to update the required fields
2541	* before using them.
2542	*
2543	* @remarks No-long-jump zone!!!
2544	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2545	*/
2546	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2547	{
2548	/*
2549	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2550	* inhibit interrupts or clear any existing interrupt-inhibition.
2551	*/
2552	uint32_t uIntrState = 0;
2553	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2554	{
2555	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2556	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2557	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2558	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2559	{
2560	/*
2561	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2562	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2563	*/
2564	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2565	}
2566	else if (pMixedCtx->eflags.Bits.u1IF)
2567	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2568	else
2569	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2570	}
2571	return uIntrState;
2572	}
2573
2574
2575	/**
2576	* Loads the guest's interruptibility-state into the guest-state area in the
2577	* VMCS.
2578	*
2579	* @returns VBox status code.
2580	* @param pVCpu Pointer to the VMCPU.
2581	* @param uIntrState The interruptibility-state to set.
2582	*/
2583	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2584	{
2585	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2586	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2587	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2588	AssertRCReturn(rc, rc);
2589	return rc;
2590	}
2591
2592
2593	/**
2594	* Loads the guest's RIP into the guest-state area in the VMCS.
2595	*
2596	* @returns VBox status code.
2597	* @param pVCpu Pointer to the VMCPU.
2598	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2599	* out-of-sync. Make sure to update the required fields
2600	* before using them.
2601	*
2602	* @remarks No-long-jump zone!!!
2603	*/
2604	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2605	{
2606	int rc = VINF_SUCCESS;
2607	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2608	{
2609	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2610	AssertRCReturn(rc, rc);
2611	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2612	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2613	}
2614	return rc;
2615	}
2616
2617
2618	/**
2619	* Loads the guest's RSP into the guest-state area in the VMCS.
2620	*
2621	* @returns VBox status code.
2622	* @param pVCpu Pointer to the VMCPU.
2623	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2624	* out-of-sync. Make sure to update the required fields
2625	* before using them.
2626	*
2627	* @remarks No-long-jump zone!!!
2628	*/
2629	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2630	{
2631	int rc = VINF_SUCCESS;
2632	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2633	{
2634	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2635	AssertRCReturn(rc, rc);
2636	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2637	}
2638	return rc;
2639	}
2640
2641
2642	/**
2643	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2644	*
2645	* @returns VBox status code.
2646	* @param pVCpu Pointer to the VMCPU.
2647	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2648	* out-of-sync. Make sure to update the required fields
2649	* before using them.
2650	*
2651	* @remarks No-long-jump zone!!!
2652	*/
2653	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2654	{
2655	int rc = VINF_SUCCESS;
2656	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2657	{
2658	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2659	Let us assert it as such and use 32-bit VMWRITE. */
2660	Assert(!(pMixedCtx->rflags.u64 >> 32));
2661	X86EFLAGS uEFlags = pMixedCtx->eflags;
2662	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2663	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2664
2665	/*
2666	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2667	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2668	*/
2669	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2670	{
2671	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2672	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2673	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2674	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2675	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2676	}
2677
2678	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2679	AssertRCReturn(rc, rc);
2680
2681	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2682	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2683	}
2684	return rc;
2685	}
2686
2687
2688	/**
2689	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2690	*
2691	* @returns VBox status code.
2692	* @param pVCpu Pointer to the VMCPU.
2693	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2694	* out-of-sync. Make sure to update the required fields
2695	* before using them.
2696	*
2697	* @remarks No-long-jump zone!!!
2698	*/
2699	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2700	{
2701	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2702	AssertRCReturn(rc, rc);
2703	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2704	AssertRCReturn(rc, rc);
2705	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2706	AssertRCReturn(rc, rc);
2707	return rc;
2708	}
2709
2710
2711	/**
2712	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2713	* in the VMCS.
2714	*
2715	* @returns VBox status code.
2716	* @param pVM Pointer to the VM.
2717	* @param pVCpu Pointer to the VMCPU.
2718	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2719	* out-of-sync. Make sure to update the required fields
2720	* before using them.
2721	*
2722	* @remarks No-long-jump zone!!!
2723	*/
2724	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2725	{
2726	int rc = VINF_SUCCESS;
2727	PVM pVM = pVCpu->CTX_SUFF(pVM);
2728
2729	/*
2730	* Guest CR0.
2731	* Guest FPU.
2732	*/
2733	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2734	{
2735	Assert(!(pCtx->cr0 >> 32));
2736	uint32_t u32GuestCR0 = pCtx->cr0;
2737
2738	/* The guest's view (read access) of its CR0 is unblemished. */
2739	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2740	AssertRCReturn(rc, rc);
2741	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2742
2743	/* Setup VT-x's view of the guest CR0. */
2744	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2745	if (pVM->hm.s.fNestedPaging)
2746	{
2747	if (CPUMIsGuestPagingEnabledEx(pCtx))
2748	{
2749	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2750	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2751	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2752	}
2753	else
2754	{
2755	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2756	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2757	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2758	}
2759
2760	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2761	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2762	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2763
2764	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2765	AssertRCReturn(rc, rc);
2766	}
2767	else
2768	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2769
2770	/*
2771	* Guest FPU bits.
2772	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2773	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2774	*/
2775	u32GuestCR0 \|= X86_CR0_NE;
2776	bool fInterceptNM = false;
2777	if (CPUMIsGuestFPUStateActive(pVCpu))
2778	{
2779	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2780	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2781	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2782	}
2783	else
2784	{
2785	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2786	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2787	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2788	}
2789
2790	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2791	bool fInterceptMF = false;
2792	if (!(pCtx->cr0 & X86_CR0_NE))
2793	fInterceptMF = true;
2794
2795	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2796	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2797	{
2798	Assert(PDMVmmDevHeapIsEnabled(pVM));
2799	Assert(pVM->hm.s.vmx.pRealModeTSS);
2800	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2801	fInterceptNM = true;
2802	fInterceptMF = true;
2803	}
2804	else
2805	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2806
2807	if (fInterceptNM)
2808	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2809	else
2810	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2811
2812	if (fInterceptMF)
2813	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2814	else
2815	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2816
2817	/* Additional intercepts for debugging, define these yourself explicitly. */
2818	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2819	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2820	\| RT_BIT(X86_XCPT_DB)
2821	\| RT_BIT(X86_XCPT_DE)
2822	\| RT_BIT(X86_XCPT_NM)
2823	\| RT_BIT(X86_XCPT_UD)
2824	\| RT_BIT(X86_XCPT_NP)
2825	\| RT_BIT(X86_XCPT_SS)
2826	\| RT_BIT(X86_XCPT_GP)
2827	\| RT_BIT(X86_XCPT_PF)
2828	\| RT_BIT(X86_XCPT_MF);
2829	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2830	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2831	#endif
2832
2833	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2834
2835	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2836	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2837	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2838	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2839	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2840	else
2841	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2842
2843	u32GuestCR0 \|= uSetCR0;
2844	u32GuestCR0 &= uZapCR0;
2845	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2846
2847	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2848	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2849	AssertRCReturn(rc, rc);
2850	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2851	AssertRCReturn(rc, rc);
2852	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2853
2854	/*
2855	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2856	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2857	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2858	*/
2859	uint32_t u32CR0Mask = 0;
2860	u32CR0Mask = X86_CR0_PE
2861	\| X86_CR0_NE
2862	\| X86_CR0_WP
2863	\| X86_CR0_PG
2864	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2865	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2866	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2867	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2868	u32CR0Mask &= ~X86_CR0_PE;
2869	if (pVM->hm.s.fNestedPaging)
2870	u32CR0Mask &= ~X86_CR0_WP;
2871
2872	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2873	if (fInterceptNM)
2874	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2875	else
2876	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2877
2878	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2879	pVCpu->hm.s.vmx.cr0_mask = u32CR0Mask;
2880	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2881	AssertRCReturn(rc, rc);
2882
2883	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2884	}
2885
2886	/*
2887	* Guest CR2.
2888	* It's always loaded in the assembler code. Nothing to do here.
2889	*/
2890
2891	/*
2892	* Guest CR3.
2893	*/
2894	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2895	{
2896	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2897	if (pVM->hm.s.fNestedPaging)
2898	{
2899	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2900
2901	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2902	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2903	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2904	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2905
2906	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2907	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2908	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2909
2910	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2911	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2912	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2913	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2914
2915	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2916	AssertRCReturn(rc, rc);
2917	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2918
2919	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2920	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2921	{
2922	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2923	if (CPUMIsGuestInPAEModeEx(pCtx))
2924	{
2925	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
2926	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
2927	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
2928	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
2929	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
2930	}
2931
2932	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2933	have Unrestricted Execution to handle the guest when it's not using paging. */
2934	GCPhysGuestCR3 = pCtx->cr3;
2935	}
2936	else
2937	{
2938	/*
2939	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2940	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2941	* EPT takes care of translating it to host-physical addresses.
2942	*/
2943	RTGCPHYS GCPhys;
2944	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2945	Assert(PDMVmmDevHeapIsEnabled(pVM));
2946
2947	/* We obtain it here every time as the guest could have relocated this PCI region. */
2948	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2949	AssertRCReturn(rc, rc);
2950
2951	GCPhysGuestCR3 = GCPhys;
2952	}
2953
2954	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
2955	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2956	}
2957	else
2958	{
2959	/* Non-nested paging case, just use the hypervisor's CR3. */
2960	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2961
2962	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
2963	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
2964	}
2965	AssertRCReturn(rc, rc);
2966
2967	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2968	}
2969
2970	/*
2971	* Guest CR4.
2972	*/
2973	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2974	{
2975	Assert(!(pCtx->cr4 >> 32));
2976	uint32_t u32GuestCR4 = pCtx->cr4;
2977
2978	/* The guest's view of its CR4 is unblemished. */
2979	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2980	AssertRCReturn(rc, rc);
2981	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2982
2983	/* Setup VT-x's view of the guest CR4. */
2984	/*
2985	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2986	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2987	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2988	*/
2989	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2990	{
2991	Assert(pVM->hm.s.vmx.pRealModeTSS);
2992	Assert(PDMVmmDevHeapIsEnabled(pVM));
2993	u32GuestCR4 &= ~X86_CR4_VME;
2994	}
2995
2996	if (pVM->hm.s.fNestedPaging)
2997	{
2998	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2999	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3000	{
3001	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3002	u32GuestCR4 \|= X86_CR4_PSE;
3003	/* Our identity mapping is a 32 bits page directory. */
3004	u32GuestCR4 &= ~X86_CR4_PAE;
3005	}
3006	/* else use guest CR4.*/
3007	}
3008	else
3009	{
3010	/*
3011	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3012	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3013	*/
3014	switch (pVCpu->hm.s.enmShadowMode)
3015	{
3016	case PGMMODE_REAL: /* Real-mode. */
3017	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3018	case PGMMODE_32_BIT: /* 32-bit paging. */
3019	{
3020	u32GuestCR4 &= ~X86_CR4_PAE;
3021	break;
3022	}
3023
3024	case PGMMODE_PAE: /* PAE paging. */
3025	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3026	{
3027	u32GuestCR4 \|= X86_CR4_PAE;
3028	break;
3029	}
3030
3031	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3032	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3033	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3034	break;
3035	#endif
3036	default:
3037	AssertFailed();
3038	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3039	}
3040	}
3041
3042	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3043	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3044	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3045	u32GuestCR4 \|= uSetCR4;
3046	u32GuestCR4 &= uZapCR4;
3047
3048	/* Write VT-x's view of the guest CR4 into the VMCS. */
3049	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3050	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3051	AssertRCReturn(rc, rc);
3052
3053	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3054	uint32_t u32CR4Mask = 0;
3055	u32CR4Mask = X86_CR4_VME
3056	\| X86_CR4_PAE
3057	\| X86_CR4_PGE
3058	\| X86_CR4_PSE
3059	\| X86_CR4_VMXE;
3060	pVCpu->hm.s.vmx.cr4_mask = u32CR4Mask;
3061	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3062	AssertRCReturn(rc, rc);
3063
3064	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3065	}
3066	return rc;
3067	}
3068
3069
3070	/**
3071	* Loads the guest debug registers into the guest-state area in the VMCS.
3072	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3073	*
3074	* @returns VBox status code.
3075	* @param pVCpu Pointer to the VMCPU.
3076	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3077	* out-of-sync. Make sure to update the required fields
3078	* before using them.
3079	*
3080	* @remarks No-long-jump zone!!!
3081	*/
3082	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3083	{
3084	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3085	return VINF_SUCCESS;
3086
3087	#ifdef VBOX_STRICT
3088	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3089	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3090	{
3091	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3092	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3093	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3094	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3095	}
3096	#endif
3097
3098	int rc = VERR_INTERNAL_ERROR_5;
3099	PVM pVM = pVCpu->CTX_SUFF(pVM);
3100	bool fInterceptDB = false;
3101	bool fInterceptMovDRx = false;
3102	if (DBGFIsStepping(pVCpu))
3103	{
3104	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3105	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3106	{
3107	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3108	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3109	AssertRCReturn(rc, rc);
3110	Assert(fInterceptDB == false);
3111	}
3112	else
3113	fInterceptDB = true;
3114	}
3115
3116	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3117	{
3118	if (!CPUMIsHyperDebugStateActive(pVCpu))
3119	{
3120	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3121	AssertRC(rc);
3122	}
3123	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3124	fInterceptMovDRx = true;
3125	}
3126	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3127	{
3128	if (!CPUMIsGuestDebugStateActive(pVCpu))
3129	{
3130	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3131	AssertRC(rc);
3132	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3133	}
3134	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3135	Assert(fInterceptMovDRx == false);
3136	}
3137	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3138	{
3139	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3140	fInterceptMovDRx = true;
3141	}
3142
3143	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3144	if (fInterceptDB)
3145	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3146	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3147	{
3148	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3149	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3150	#endif
3151	}
3152
3153	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3154	if (fInterceptMovDRx)
3155	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3156	else
3157	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3158
3159	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3160	AssertRCReturn(rc, rc);
3161	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3162	AssertRCReturn(rc, rc);
3163
3164	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3165	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3166	AssertRCReturn(rc, rc);
3167
3168	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3169	return rc;
3170	}
3171
3172
3173	#ifdef VBOX_STRICT
3174	/**
3175	* Strict function to validate segment registers.
3176	*
3177	* @remarks Requires CR0.
3178	*/
3179	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3180	{
3181	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3182	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3183	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3184
3185	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3186	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3187	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3188	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3189	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3190	{
3191	/* Protected mode checks */
3192	/* CS */
3193	Assert(pCtx->cs.Attr.n.u1Present);
3194	Assert(!(pCtx->cs.Attr.u & 0xf00));
3195	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3196	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3197	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3198	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3199	\|\| (pCtx->cs.Attr.n.u1Granularity));
3200	/* CS cannot be loaded with NULL in protected mode. */
3201	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3202	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3203	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3204	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3205	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3206	else
3207	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3208	/* SS */
3209	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3210	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3211	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3212	if ( !(pCtx->cr0 & X86_CR0_PE)
3213	\|\| pCtx->cs.Attr.n.u4Type == 3)
3214	{
3215	Assert(!pCtx->ss.Attr.n.u2Dpl);
3216	}
3217	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3218	{
3219	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3220	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3221	Assert(pCtx->ss.Attr.n.u1Present);
3222	Assert(!(pCtx->ss.Attr.u & 0xf00));
3223	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3224	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3225	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3226	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3227	\|\| (pCtx->ss.Attr.n.u1Granularity));
3228	}
3229	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3230	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3231	{
3232	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3233	Assert(pCtx->ds.Attr.n.u1Present);
3234	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3235	Assert(!(pCtx->ds.Attr.u & 0xf00));
3236	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3237	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3238	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3239	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3240	\|\| (pCtx->ds.Attr.n.u1Granularity));
3241	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3242	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3243	}
3244	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3245	{
3246	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3247	Assert(pCtx->es.Attr.n.u1Present);
3248	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3249	Assert(!(pCtx->es.Attr.u & 0xf00));
3250	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3251	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3252	\|\| !(pCtx->es.Attr.n.u1Granularity));
3253	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3254	\|\| (pCtx->es.Attr.n.u1Granularity));
3255	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3256	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3257	}
3258	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3259	{
3260	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3261	Assert(pCtx->fs.Attr.n.u1Present);
3262	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3263	Assert(!(pCtx->fs.Attr.u & 0xf00));
3264	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3265	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3266	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3267	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3268	\|\| (pCtx->fs.Attr.n.u1Granularity));
3269	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3270	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3271	}
3272	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3273	{
3274	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3275	Assert(pCtx->gs.Attr.n.u1Present);
3276	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3277	Assert(!(pCtx->gs.Attr.u & 0xf00));
3278	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3279	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3280	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3281	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3282	\|\| (pCtx->gs.Attr.n.u1Granularity));
3283	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3284	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3285	}
3286	/* 64-bit capable CPUs. */
3287	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3288	Assert(!(pCtx->cs.u64Base >> 32));
3289	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3290	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3291	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3292	# endif
3293	}
3294	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3295	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3296	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3297	{
3298	/* Real and v86 mode checks. */
3299	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3300	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3301	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3302	{
3303	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3304	}
3305	else
3306	{
3307	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3308	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3309	}
3310
3311	/* CS */
3312	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3313	Assert(pCtx->cs.u32Limit == 0xffff);
3314	Assert(u32CSAttr == 0xf3);
3315	/* SS */
3316	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3317	Assert(pCtx->ss.u32Limit == 0xffff);
3318	Assert(u32SSAttr == 0xf3);
3319	/* DS */
3320	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3321	Assert(pCtx->ds.u32Limit == 0xffff);
3322	Assert(u32DSAttr == 0xf3);
3323	/* ES */
3324	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3325	Assert(pCtx->es.u32Limit == 0xffff);
3326	Assert(u32ESAttr == 0xf3);
3327	/* FS */
3328	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3329	Assert(pCtx->fs.u32Limit == 0xffff);
3330	Assert(u32FSAttr == 0xf3);
3331	/* GS */
3332	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3333	Assert(pCtx->gs.u32Limit == 0xffff);
3334	Assert(u32GSAttr == 0xf3);
3335	/* 64-bit capable CPUs. */
3336	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3337	Assert(!(pCtx->cs.u64Base >> 32));
3338	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3339	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3340	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3341	# endif
3342	}
3343	}
3344	#endif /* VBOX_STRICT */
3345
3346
3347	/**
3348	* Writes a guest segment register into the guest-state area in the VMCS.
3349	*
3350	* @returns VBox status code.
3351	* @param pVCpu Pointer to the VMCPU.
3352	* @param idxSel Index of the selector in the VMCS.
3353	* @param idxLimit Index of the segment limit in the VMCS.
3354	* @param idxBase Index of the segment base in the VMCS.
3355	* @param idxAccess Index of the access rights of the segment in the VMCS.
3356	* @param pSelReg Pointer to the segment selector.
3357	* @param pCtx Pointer to the guest-CPU context.
3358	*
3359	* @remarks No-long-jump zone!!!
3360	*/
3361	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3362	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3363	{
3364	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3365	AssertRCReturn(rc, rc);
3366	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3367	AssertRCReturn(rc, rc);
3368	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3369	AssertRCReturn(rc, rc);
3370
3371	uint32_t u32Access = pSelReg->Attr.u;
3372	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3373	{
3374	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3375	u32Access = 0xf3;
3376	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3377	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3378	}
3379	else
3380	{
3381	/*
3382	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3383	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3384	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3385	* loaded in protected-mode have their attribute as 0.
3386	*/
3387	if (!u32Access)
3388	u32Access = HMVMX_SEL_UNUSABLE;
3389	}
3390
3391	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3392	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3393	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3394
3395	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3396	AssertRCReturn(rc, rc);
3397	return rc;
3398	}
3399
3400
3401	/**
3402	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3403	* into the guest-state area in the VMCS.
3404	*
3405	* @returns VBox status code.
3406	* @param pVM Pointer to the VM.
3407	* @param pVCPU Pointer to the VMCPU.
3408	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3409	* out-of-sync. Make sure to update the required fields
3410	* before using them.
3411	*
3412	* @remarks Requires CR0 (strict builds validation).
3413	* @remarks No-long-jump zone!!!
3414	*/
3415	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3416	{
3417	int rc = VERR_INTERNAL_ERROR_5;
3418	PVM pVM = pVCpu->CTX_SUFF(pVM);
3419
3420	/*
3421	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3422	*/
3423	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3424	{
3425	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3426	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3427	{
3428	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3429	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3430	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3431	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3432	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3433	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3434	}
3435
3436	#ifdef VBOX_WITH_REM
3437	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3438	{
3439	Assert(pVM->hm.s.vmx.pRealModeTSS);
3440	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3441	if ( pVCpu->hm.s.vmx.fWasInRealMode
3442	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3443	{
3444	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3445	in real-mode (e.g. OpenBSD 4.0) */
3446	REMFlushTBs(pVM);
3447	Log(("Load: Switch to protected mode detected!\n"));
3448	pVCpu->hm.s.vmx.fWasInRealMode = false;
3449	}
3450	}
3451	#endif
3452	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3453	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3454	AssertRCReturn(rc, rc);
3455	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3456	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3457	AssertRCReturn(rc, rc);
3458	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3459	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3460	AssertRCReturn(rc, rc);
3461	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3462	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3463	AssertRCReturn(rc, rc);
3464	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3465	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3466	AssertRCReturn(rc, rc);
3467	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3468	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3469	AssertRCReturn(rc, rc);
3470
3471	Log(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3472	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3473	#ifdef VBOX_STRICT
3474	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3475	#endif
3476	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3477	}
3478
3479	/*
3480	* Guest TR.
3481	*/
3482	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3483	{
3484	/*
3485	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3486	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3487	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3488	*/
3489	uint16_t u16Sel = 0;
3490	uint32_t u32Limit = 0;
3491	uint64_t u64Base = 0;
3492	uint32_t u32AccessRights = 0;
3493
3494	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3495	{
3496	u16Sel = pMixedCtx->tr.Sel;
3497	u32Limit = pMixedCtx->tr.u32Limit;
3498	u64Base = pMixedCtx->tr.u64Base;
3499	u32AccessRights = pMixedCtx->tr.Attr.u;
3500	}
3501	else
3502	{
3503	Assert(pVM->hm.s.vmx.pRealModeTSS);
3504	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3505
3506	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3507	RTGCPHYS GCPhys;
3508	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3509	AssertRCReturn(rc, rc);
3510
3511	X86DESCATTR DescAttr;
3512	DescAttr.u = 0;
3513	DescAttr.n.u1Present = 1;
3514	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3515
3516	u16Sel = 0;
3517	u32Limit = HM_VTX_TSS_SIZE;
3518	u64Base = GCPhys; /* in real-mode phys = virt. */
3519	u32AccessRights = DescAttr.u;
3520	}
3521
3522	/* Validate. */
3523	Assert(!(u16Sel & RT_BIT(2)));
3524	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3525	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3526	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3527	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3528	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3529	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3530	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3531	Assert( (u32Limit & 0xfff) == 0xfff
3532	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3533	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3534	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3535
3536	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3537	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3538	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3539	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3540
3541	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3542	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3543	}
3544
3545	/*
3546	* Guest GDTR.
3547	*/
3548	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3549	{
3550	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3551	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3552
3553	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3554	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3555	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3556	}
3557
3558	/*
3559	* Guest LDTR.
3560	*/
3561	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3562	{
3563	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3564	uint32_t u32Access = 0;
3565	if (!pMixedCtx->ldtr.Attr.u)
3566	u32Access = HMVMX_SEL_UNUSABLE;
3567	else
3568	u32Access = pMixedCtx->ldtr.Attr.u;
3569
3570	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3571	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3572	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3573	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3574
3575	/* Validate. */
3576	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3577	{
3578	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3579	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3580	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3581	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3582	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3583	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3584	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3585	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3586	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3587	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3588	}
3589
3590	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3591	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3592	}
3593
3594	/*
3595	* Guest IDTR.
3596	*/
3597	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3598	{
3599	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3600	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3601
3602	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3603	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3604	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3605	}
3606
3607	return VINF_SUCCESS;
3608	}
3609
3610
3611	/**
3612	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3613	* areas. These MSRs will automatically be loaded to the host CPU on every
3614	* successful VM entry and stored from the host CPU on every successful VM exit.
3615	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3616	*
3617	* @returns VBox status code.
3618	* @param pVCpu Pointer to the VMCPU.
3619	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3620	* out-of-sync. Make sure to update the required fields
3621	* before using them.
3622	*
3623	* @remarks No-long-jump zone!!!
3624	*/
3625	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3626	{
3627	AssertPtr(pVCpu);
3628	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3629
3630	/*
3631	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3632	*/
3633	int rc = VINF_SUCCESS;
3634	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3635	{
3636	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3637	PVM pVM = pVCpu->CTX_SUFF(pVM);
3638	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3639	uint32_t cGuestMsrs = 0;
3640
3641	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3642	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3643	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3644	if (fSupportsNX \|\| fSupportsLongMode)
3645	{
3646	/** @todo support save IA32_EFER, i.e.
3647	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, in which case the
3648	* guest EFER need not be part of the VM-entry MSR-load area. */
3649	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3650	pGuestMsr->u32Reserved = 0;
3651	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3652	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3653	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3654	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3655	pGuestMsr++; cGuestMsrs++;
3656	if (fSupportsLongMode)
3657	{
3658	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3659	pGuestMsr->u32Reserved = 0;
3660	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3661	pGuestMsr++; cGuestMsrs++;
3662	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3663	pGuestMsr->u32Reserved = 0;
3664	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3665	pGuestMsr++; cGuestMsrs++;
3666	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3667	pGuestMsr->u32Reserved = 0;
3668	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3669	pGuestMsr++; cGuestMsrs++;
3670	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3671	pGuestMsr->u32Reserved = 0;
3672	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3673	pGuestMsr++; cGuestMsrs++;
3674	}
3675	}
3676
3677	/*
3678	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3679	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3680	*/
3681	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3682	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3683	{
3684	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3685	pGuestMsr->u32Reserved = 0;
3686	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3687	AssertRCReturn(rc, rc);
3688	pGuestMsr++; cGuestMsrs++;
3689	}
3690
3691	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3692	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3693	{
3694	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3695	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3696	}
3697
3698	/* Update the VCPU's copy of the guest MSR count. */
3699	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3700	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3701	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3702	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3703
3704	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3705	}
3706
3707	/*
3708	* Guest Sysenter MSRs.
3709	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3710	* VM-exits on WRMSRs for these MSRs.
3711	*/
3712	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3713	{
3714	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3715	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3716	}
3717	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3718	{
3719	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3720	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3721	}
3722	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3723	{
3724	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3725	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3726	}
3727
3728	return rc;
3729	}
3730
3731
3732	/**
3733	* Loads the guest activity state into the guest-state area in the VMCS.
3734	*
3735	* @returns VBox status code.
3736	* @param pVCpu Pointer to the VMCPU.
3737	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3738	* out-of-sync. Make sure to update the required fields
3739	* before using them.
3740	*
3741	* @remarks No-long-jump zone!!!
3742	*/
3743	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3744	{
3745	/** @todo See if we can make use of other states, e.g.
3746	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3747	int rc = VINF_SUCCESS;
3748	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3749	{
3750	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3751	AssertRCReturn(rc, rc);
3752	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3753	}
3754	return rc;
3755	}
3756
3757
3758	/**
3759	* Sets up the appropriate function to run guest code.
3760	*
3761	* @returns VBox status code.
3762	* @param pVCpu Pointer to the VMCPU.
3763	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3764	* out-of-sync. Make sure to update the required fields
3765	* before using them.
3766	*
3767	* @remarks No-long-jump zone!!!
3768	*/
3769	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3770	{
3771	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3772	{
3773	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3774	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3775	#endif
3776	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3777	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3778	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3779	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3780	#else
3781	/* 64-bit host or hybrid host. */
3782	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3783	#endif
3784	}
3785	else
3786	{
3787	/* Guest is not in long mode, use the 32-bit handler. */
3788	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3789	}
3790	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3791	return VINF_SUCCESS;
3792	}
3793
3794
3795	/**
3796	* Wrapper for running the guest code in VT-x.
3797	*
3798	* @returns VBox strict status code.
3799	* @param pVM Pointer to the VM.
3800	* @param pVCpu Pointer to the VMCPU.
3801	* @param pCtx Pointer to the guest-CPU context.
3802	*
3803	* @remarks No-long-jump zone!!!
3804	*/
3805	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3806	{
3807	/*
3808	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3809	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3810	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3811	*/
3812	#ifdef VBOX_WITH_KERNEL_USING_XMM
3813	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3814	#else
3815	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3816	#endif
3817	}
3818
3819
3820	/**
3821	* Report world-switch error and dump some useful debug info.
3822	*
3823	* @param pVM Pointer to the VM.
3824	* @param pVCpu Pointer to the VMCPU.
3825	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3826	* @param pCtx Pointer to the guest-CPU context.
3827	* @param pVmxTransient Pointer to the VMX transient structure (only
3828	* exitReason updated).
3829	*/
3830	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3831	{
3832	Assert(pVM);
3833	Assert(pVCpu);
3834	Assert(pCtx);
3835	Assert(pVmxTransient);
3836	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3837
3838	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3839	switch (rcVMRun)
3840	{
3841	case VERR_VMX_INVALID_VMXON_PTR:
3842	AssertFailed();
3843	break;
3844	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3845	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3846	{
3847	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3848	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3849	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3850	AssertRC(rc);
3851
3852	#ifdef VBOX_STRICT
3853	Log(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3854	pVmxTransient->uExitReason));
3855	Log(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3856	Log(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3857	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3858	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3859	else
3860	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3861
3862	/* VMX control bits. */
3863	uint32_t u32Val;
3864	uint64_t u64Val;
3865	HMVMXHCUINTREG uHCReg;
3866	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3867	Log(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3868	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3869	Log(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3870	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3871	Log(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3872	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3873	Log(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3874	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3875	Log(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3876	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3877	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3878	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3879	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3880	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3881	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3882	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3883	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3884	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3885	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3886	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3887	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3888	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3889	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3890	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3891	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3892	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3893	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3894	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3895	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3896	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3897	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3898	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3899	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3900	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3901	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3902	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3903	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3904	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3905	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3906	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3907	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3908
3909	/* Guest bits. */
3910	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
3911	Log(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
3912	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
3913	Log(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
3914	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3915	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3916	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3917	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3918
3919	/* Host bits. */
3920	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3921	Log(("Host CR0 %#RHr\n", uHCReg));
3922	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3923	Log(("Host CR3 %#RHr\n", uHCReg));
3924	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3925	Log(("Host CR4 %#RHr\n", uHCReg));
3926
3927	RTGDTR HostGdtr;
3928	PCX86DESCHC pDesc;
3929	ASMGetGDTR(&HostGdtr);
3930	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
3931	Log(("Host CS %#08x\n", u32Val));
3932	if (u32Val < HostGdtr.cbGdt)
3933	{
3934	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3935	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3936	}
3937
3938	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3939	Log(("Host DS %#08x\n", u32Val));
3940	if (u32Val < HostGdtr.cbGdt)
3941	{
3942	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3943	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3944	}
3945
3946	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3947	Log(("Host ES %#08x\n", u32Val));
3948	if (u32Val < HostGdtr.cbGdt)
3949	{
3950	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3951	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3952	}
3953
3954	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3955	Log(("Host FS %#08x\n", u32Val));
3956	if (u32Val < HostGdtr.cbGdt)
3957	{
3958	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3959	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3960	}
3961
3962	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3963	Log(("Host GS %#08x\n", u32Val));
3964	if (u32Val < HostGdtr.cbGdt)
3965	{
3966	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3967	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3968	}
3969
3970	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3971	Log(("Host SS %#08x\n", u32Val));
3972	if (u32Val < HostGdtr.cbGdt)
3973	{
3974	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3975	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3976	}
3977
3978	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3979	Log(("Host TR %#08x\n", u32Val));
3980	if (u32Val < HostGdtr.cbGdt)
3981	{
3982	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3983	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3984	}
3985
3986	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3987	Log(("Host TR Base %#RHv\n", uHCReg));
3988	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3989	Log(("Host GDTR Base %#RHv\n", uHCReg));
3990	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3991	Log(("Host IDTR Base %#RHv\n", uHCReg));
3992	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3993	Log(("Host SYSENTER CS %#08x\n", u32Val));
3994	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3995	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3996	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3997	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3998	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3999	Log(("Host RSP %#RHv\n", uHCReg));
4000	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4001	Log(("Host RIP %#RHv\n", uHCReg));
4002	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4003	if (HMVMX_IS_64BIT_HOST_MODE())
4004	{
4005	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4006	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4007	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4008	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4009	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4010	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4011	}
4012	# endif
4013	#endif /* VBOX_STRICT */
4014	break;
4015	}
4016
4017	default:
4018	/* Impossible */
4019	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4020	break;
4021	}
4022	NOREF(pVM);
4023	}
4024
4025
4026	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4027	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4028	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4029	#endif
4030	#ifdef VBOX_STRICT
4031	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4032	{
4033	switch (idxField)
4034	{
4035	case VMX_VMCS_GUEST_RIP:
4036	case VMX_VMCS_GUEST_RSP:
4037	case VMX_VMCS_GUEST_SYSENTER_EIP:
4038	case VMX_VMCS_GUEST_SYSENTER_ESP:
4039	case VMX_VMCS_GUEST_GDTR_BASE:
4040	case VMX_VMCS_GUEST_IDTR_BASE:
4041	case VMX_VMCS_GUEST_CS_BASE:
4042	case VMX_VMCS_GUEST_DS_BASE:
4043	case VMX_VMCS_GUEST_ES_BASE:
4044	case VMX_VMCS_GUEST_FS_BASE:
4045	case VMX_VMCS_GUEST_GS_BASE:
4046	case VMX_VMCS_GUEST_SS_BASE:
4047	case VMX_VMCS_GUEST_LDTR_BASE:
4048	case VMX_VMCS_GUEST_TR_BASE:
4049	case VMX_VMCS_GUEST_CR3:
4050	return true;
4051	}
4052	return false;
4053	}
4054
4055	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4056	{
4057	switch (idxField)
4058	{
4059	/* Read-only fields. */
4060	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4061	return true;
4062	}
4063	/* Remaining readable fields should also be writable. */
4064	return hmR0VmxIsValidWriteField(idxField);
4065	}
4066	#endif /* VBOX_STRICT */
4067
4068
4069	/**
4070	* Executes the specified handler in 64-bit mode.
4071	*
4072	* @returns VBox status code.
4073	* @param pVM Pointer to the VM.
4074	* @param pVCpu Pointer to the VMCPU.
4075	* @param pCtx Pointer to the guest CPU context.
4076	* @param enmOp The operation to perform.
4077	* @param cbParam Number of parameters.
4078	* @param paParam Array of 32-bit parameters.
4079	*/
4080	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4081	uint32_t *paParam)
4082	{
4083	int rc, rc2;
4084	PHMGLOBLCPUINFO pCpu;
4085	RTHCPHYS HCPhysCpuPage;
4086	RTCCUINTREG uOldEFlags;
4087
4088	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4089	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4090	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4091	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4092
4093	#ifdef VBOX_STRICT
4094	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4095	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4096
4097	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4098	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4099	#endif
4100
4101	/* Disable interrupts. */
4102	uOldEFlags = ASMIntDisableFlags();
4103
4104	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4105	RTCPUID idHostCpu = RTMpCpuId();
4106	CPUMR0SetLApic(pVM, idHostCpu);
4107	#endif
4108
4109	pCpu = HMR0GetCurrentCpu();
4110	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4111
4112	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4113	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4114
4115	/* Leave VMX Root Mode. */
4116	VMXDisable();
4117
4118	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4119
4120	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4121	CPUMSetHyperEIP(pVCpu, enmOp);
4122	for (int i = (int)cbParam - 1; i >= 0; i--)
4123	CPUMPushHyper(pVCpu, paParam[i]);
4124
4125	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4126
4127	/* Call the switcher. */
4128	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4129	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4130
4131	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4132	/* Make sure the VMX instructions don't cause #UD faults. */
4133	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4134
4135	/* Re-enter VMX Root Mode */
4136	rc2 = VMXEnable(HCPhysCpuPage);
4137	if (RT_FAILURE(rc2))
4138	{
4139	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4140	ASMSetFlags(uOldEFlags);
4141	return rc2;
4142	}
4143
4144	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4145	AssertRC(rc2);
4146	Assert(!(ASMGetFlags() & X86_EFL_IF));
4147	ASMSetFlags(uOldEFlags);
4148	return rc;
4149	}
4150
4151
4152	/**
4153	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4154	* supporting 64-bit guests.
4155	*
4156	* @returns VBox status code.
4157	* @param fResume Whether to VMLAUNCH or VMRESUME.
4158	* @param pCtx Pointer to the guest-CPU context.
4159	* @param pCache Pointer to the VMCS cache.
4160	* @param pVM Pointer to the VM.
4161	* @param pVCpu Pointer to the VMCPU.
4162	*/
4163	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4164	{
4165	uint32_t aParam[6];
4166	PHMGLOBLCPUINFO pCpu = NULL;
4167	RTHCPHYS HCPhysCpuPage = 0;
4168	int rc = VERR_INTERNAL_ERROR_5;
4169
4170	pCpu = HMR0GetCurrentCpu();
4171	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4172
4173	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4174	pCache->uPos = 1;
4175	pCache->interPD = PGMGetInterPaeCR3(pVM);
4176	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4177	#endif
4178
4179	#ifdef VBOX_STRICT
4180	pCache->TestIn.HCPhysCpuPage = 0;
4181	pCache->TestIn.HCPhysVmcs = 0;
4182	pCache->TestIn.pCache = 0;
4183	pCache->TestOut.HCPhysVmcs = 0;
4184	pCache->TestOut.pCache = 0;
4185	pCache->TestOut.pCtx = 0;
4186	pCache->TestOut.eflags = 0;
4187	#endif
4188
4189	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4190	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4191	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4192	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4193	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4194	aParam[5] = 0;
4195
4196	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4197	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4198	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4199	#endif
4200	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4201
4202	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4203	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4204	Assert(pCtx->dr[4] == 10);
4205	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4206	#endif
4207
4208	#ifdef VBOX_STRICT
4209	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4210	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4211	pVCpu->hm.s.vmx.HCPhysVmcs));
4212	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4213	pCache->TestOut.HCPhysVmcs));
4214	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4215	pCache->TestOut.pCache));
4216	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4217	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4218	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4219	pCache->TestOut.pCtx));
4220	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4221	#endif
4222	return rc;
4223	}
4224
4225
4226	/**
4227	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4228	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4229	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4230	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4231	*
4232	* @returns VBox status code.
4233	* @param pVM Pointer to the VM.
4234	* @param pVCpu Pointer to the VMCPU.
4235	*/
4236	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4237	{
4238	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4239	{ \
4240	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4241	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4242	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4243	++cReadFields; \
4244	}
4245
4246	AssertPtr(pVM);
4247	AssertPtr(pVCpu);
4248	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4249	uint32_t cReadFields = 0;
4250
4251	/*
4252	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4253	* and serve to indicate exceptions to the rules.
4254	*/
4255
4256	/* Guest-natural selector base fields. */
4257	#if 0
4258	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4261	#endif
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4274	#if 0
4275	/* Unused natural width guest-state fields. */
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4278	#endif
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4281
4282	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4283	#if 0
4284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4293	#endif
4294
4295	/* Natural width guest-state fields. */
4296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4297	#if 0
4298	/* Currently unused field. */
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4300	#endif
4301
4302	if (pVM->hm.s.fNestedPaging)
4303	{
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4305	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4306	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4307	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4308	}
4309	else
4310	{
4311	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4312	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4313	}
4314
4315	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4316	return VINF_SUCCESS;
4317	}
4318
4319
4320	/**
4321	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4322	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4323	* darwin, running 64-bit guests).
4324	*
4325	* @returns VBox status code.
4326	* @param pVCpu Pointer to the VMCPU.
4327	* @param idxField The VMCS field encoding.
4328	* @param u64Val 16, 32 or 64 bits value.
4329	*/
4330	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4331	{
4332	int rc;
4333	switch (idxField)
4334	{
4335	/*
4336	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4337	*/
4338	/* 64-bit Control fields. */
4339	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4340	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4341	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4342	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4343	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4344	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4345	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4346	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4347	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4348	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4349	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4350	case VMX_VMCS64_CTRL_EPTP_FULL:
4351	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4352	/* 64-bit Guest-state fields. */
4353	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4354	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4355	case VMX_VMCS64_GUEST_PAT_FULL:
4356	case VMX_VMCS64_GUEST_EFER_FULL:
4357	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4358	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4359	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4360	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4361	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4362	/* 64-bit Host-state fields. */
4363	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4364	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4365	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4366	{
4367	rc = VMXWriteVmcs32(idxField, u64Val);
4368	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4369	break;
4370	}
4371
4372	/*
4373	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4374	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4375	*/
4376	/* Natural-width Guest-state fields. */
4377	case VMX_VMCS_GUEST_CR3:
4378	case VMX_VMCS_GUEST_ES_BASE:
4379	case VMX_VMCS_GUEST_CS_BASE:
4380	case VMX_VMCS_GUEST_SS_BASE:
4381	case VMX_VMCS_GUEST_DS_BASE:
4382	case VMX_VMCS_GUEST_FS_BASE:
4383	case VMX_VMCS_GUEST_GS_BASE:
4384	case VMX_VMCS_GUEST_LDTR_BASE:
4385	case VMX_VMCS_GUEST_TR_BASE:
4386	case VMX_VMCS_GUEST_GDTR_BASE:
4387	case VMX_VMCS_GUEST_IDTR_BASE:
4388	case VMX_VMCS_GUEST_RSP:
4389	case VMX_VMCS_GUEST_RIP:
4390	case VMX_VMCS_GUEST_SYSENTER_ESP:
4391	case VMX_VMCS_GUEST_SYSENTER_EIP:
4392	{
4393	if (!(u64Val >> 32))
4394	{
4395	/* If this field is 64-bit, VT-x will zero out the top bits. */
4396	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4397	}
4398	else
4399	{
4400	/* Assert that only the 32->64 switcher case should ever come here. */
4401	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4402	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4403	}
4404	break;
4405	}
4406
4407	default:
4408	{
4409	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4410	rc = VERR_INVALID_PARAMETER;
4411	break;
4412	}
4413	}
4414	AssertRCReturn(rc, rc);
4415	return rc;
4416	}
4417
4418
4419	/**
4420	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4421	* hosts (except darwin) for 64-bit guests.
4422	*
4423	* @param pVCpu Pointer to the VMCPU.
4424	* @param idxField The VMCS field encoding.
4425	* @param u64Val 16, 32 or 64 bits value.
4426	*/
4427	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4428	{
4429	AssertPtr(pVCpu);
4430	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4431
4432	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4433	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4434
4435	/* Make sure there are no duplicates. */
4436	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4437	{
4438	if (pCache->Write.aField[i] == idxField)
4439	{
4440	pCache->Write.aFieldVal[i] = u64Val;
4441	return VINF_SUCCESS;
4442	}
4443	}
4444
4445	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4446	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4447	pCache->Write.cValidEntries++;
4448	return VINF_SUCCESS;
4449	}
4450
4451	/* Enable later when the assembly code uses these as callbacks. */
4452	#if 0
4453	/*
4454	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4455	*
4456	* @param pVCpu Pointer to the VMCPU.
4457	* @param pCache Pointer to the VMCS cache.
4458	*
4459	* @remarks No-long-jump zone!!!
4460	*/
4461	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4462	{
4463	AssertPtr(pCache);
4464	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4465	{
4466	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4467	AssertRC(rc);
4468	}
4469	pCache->Write.cValidEntries = 0;
4470	}
4471
4472
4473	/**
4474	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4475	*
4476	* @param pVCpu Pointer to the VMCPU.
4477	* @param pCache Pointer to the VMCS cache.
4478	*
4479	* @remarks No-long-jump zone!!!
4480	*/
4481	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4482	{
4483	AssertPtr(pCache);
4484	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4485	{
4486	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4487	AssertRC(rc);
4488	}
4489	}
4490	#endif
4491	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4492
4493
4494	/**
4495	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4496	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4497	* timer.
4498	*
4499	* @returns VBox status code.
4500	* @param pVCpu Pointer to the VMCPU.
4501	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4502	* out-of-sync. Make sure to update the required fields
4503	* before using them.
4504	* @remarks No-long-jump zone!!!
4505	*/
4506	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4507	{
4508	int rc = VERR_INTERNAL_ERROR_5;
4509	bool fOffsettedTsc = false;
4510	PVM pVM = pVCpu->CTX_SUFF(pVM);
4511	if (pVM->hm.s.vmx.fUsePreemptTimer)
4512	{
4513	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4514
4515	/* Make sure the returned values have sane upper and lower boundaries. */
4516	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4517	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4518	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4519	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4520
4521	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4522	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4523	}
4524	else
4525	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4526
4527	if (fOffsettedTsc)
4528	{
4529	uint64_t u64CurTSC = ASMReadTSC();
4530	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4531	{
4532	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4533	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4534
4535	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4536	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4537	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4538	}
4539	else
4540	{
4541	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4542	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4543	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4544	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4545	}
4546	}
4547	else
4548	{
4549	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4550	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4551	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4552	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4553	}
4554	}
4555
4556
4557	/**
4558	* Determines if an exception is a contributory exception. Contributory
4559	* exceptions are ones which can cause double-faults. Page-fault is
4560	* intentionally not included here as it's a conditional contributory exception.
4561	*
4562	* @returns true if the exception is contributory, false otherwise.
4563	* @param uVector The exception vector.
4564	*/
4565	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4566	{
4567	switch (uVector)
4568	{
4569	case X86_XCPT_GP:
4570	case X86_XCPT_SS:
4571	case X86_XCPT_NP:
4572	case X86_XCPT_TS:
4573	case X86_XCPT_DE:
4574	return true;
4575	default:
4576	break;
4577	}
4578	return false;
4579	}
4580
4581
4582	/**
4583	* Sets an event as a pending event to be injected into the guest.
4584	*
4585	* @param pVCpu Pointer to the VMCPU.
4586	* @param u32IntrInfo The VM-entry interruption-information field.
4587	* @param cbInstr The VM-entry instruction length in bytes (for software
4588	* interrupts, exceptions and privileged software
4589	* exceptions).
4590	* @param u32ErrCode The VM-entry exception error code.
4591	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4592	* page-fault.
4593	*/
4594	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4595	RTGCUINTPTR GCPtrFaultAddress)
4596	{
4597	Assert(!pVCpu->hm.s.Event.fPending);
4598	pVCpu->hm.s.Event.fPending = true;
4599	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4600	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4601	pVCpu->hm.s.Event.cbInstr = cbInstr;
4602	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4603	}
4604
4605
4606	/**
4607	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4608	*
4609	* @param pVCpu Pointer to the VMCPU.
4610	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4611	* out-of-sync. Make sure to update the required fields
4612	* before using them.
4613	*/
4614	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4615	{
4616	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4617	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4618	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4619	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4620	}
4621
4622
4623	/**
4624	* Handle a condition that occurred while delivering an event through the guest
4625	* IDT.
4626	*
4627	* @returns VBox status code (informational error codes included).
4628	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4629	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4630	* continue execution of the guest which will delivery the #DF.
4631	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4632	*
4633	* @param pVCpu Pointer to the VMCPU.
4634	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4635	* out-of-sync. Make sure to update the required fields
4636	* before using them.
4637	* @param pVmxTransient Pointer to the VMX transient structure.
4638	*
4639	* @remarks No-long-jump zone!!!
4640	*/
4641	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4642	{
4643	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4644	AssertRC(rc);
4645	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4646	{
4647	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4648	AssertRCReturn(rc, rc);
4649
4650	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4651	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4652	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4653
4654	typedef enum
4655	{
4656	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4657	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4658	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4659	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4660	} VMXREFLECTXCPT;
4661
4662	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4663	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4664	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4665	{
4666	enmReflect = VMXREFLECTXCPT_XCPT;
4667	#ifdef VBOX_STRICT
4668	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4669	&& uExitVector == X86_XCPT_PF)
4670	{
4671	Log(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4672	}
4673	#endif
4674	if ( uExitVector == X86_XCPT_PF
4675	&& uIdtVector == X86_XCPT_PF)
4676	{
4677	pVmxTransient->fVectoringPF = true;
4678	Log(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4679	}
4680	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4681	&& hmR0VmxIsContributoryXcpt(uExitVector)
4682	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4683	\|\| uIdtVector == X86_XCPT_PF))
4684	{
4685	enmReflect = VMXREFLECTXCPT_DF;
4686	}
4687	else if (uIdtVector == X86_XCPT_DF)
4688	enmReflect = VMXREFLECTXCPT_TF;
4689	}
4690	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4691	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4692	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4693	{
4694	/*
4695	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4696	* (whatever they are) as they reoccur when restarting the instruction.
4697	*/
4698	enmReflect = VMXREFLECTXCPT_XCPT;
4699	}
4700
4701	switch (enmReflect)
4702	{
4703	case VMXREFLECTXCPT_XCPT:
4704	{
4705	uint32_t u32ErrCode = 0;
4706	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4707	{
4708	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4709	AssertRCReturn(rc, rc);
4710	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4711	}
4712
4713	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4714	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4715	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4716	rc = VINF_SUCCESS;
4717	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4718	pVCpu->hm.s.Event.u32ErrCode));
4719	break;
4720	}
4721
4722	case VMXREFLECTXCPT_DF:
4723	{
4724	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4725	rc = VINF_VMX_DOUBLE_FAULT;
4726	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4727	uIdtVector, uExitVector));
4728	break;
4729	}
4730
4731	case VMXREFLECTXCPT_TF:
4732	{
4733	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4734	rc = VINF_EM_RESET;
4735	break;
4736	}
4737
4738	default:
4739	Assert(rc == VINF_SUCCESS);
4740	break;
4741	}
4742	}
4743	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4744	return rc;
4745	}
4746
4747
4748	/**
4749	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4750	*
4751	* @returns VBox status code.
4752	* @param pVCpu Pointer to the VMCPU.
4753	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4754	* out-of-sync. Make sure to update the required fields
4755	* before using them.
4756	*
4757	* @remarks No-long-jump zone!!!
4758	*/
4759	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4760	{
4761	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4762	{
4763	uint32_t uVal = 0;
4764	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4765	AssertRCReturn(rc, rc);
4766	uint32_t uShadow = 0;
4767	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4768	AssertRCReturn(rc, rc);
4769
4770	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4771	CPUMSetGuestCR0(pVCpu, uVal);
4772	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4773	}
4774	return VINF_SUCCESS;
4775	}
4776
4777
4778	/**
4779	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4780	*
4781	* @returns VBox status code.
4782	* @param pVCpu Pointer to the VMCPU.
4783	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4784	* out-of-sync. Make sure to update the required fields
4785	* before using them.
4786	*
4787	* @remarks No-long-jump zone!!!
4788	*/
4789	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4790	{
4791	int rc = VINF_SUCCESS;
4792	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4793	{
4794	uint32_t uVal = 0;
4795	uint32_t uShadow = 0;
4796	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4797	AssertRCReturn(rc, rc);
4798	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4799	AssertRCReturn(rc, rc);
4800
4801	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4802	CPUMSetGuestCR4(pVCpu, uVal);
4803	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4804	}
4805	return rc;
4806	}
4807
4808
4809	/**
4810	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4811	*
4812	* @returns VBox status code.
4813	* @param pVCpu Pointer to the VMCPU.
4814	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4815	* out-of-sync. Make sure to update the required fields
4816	* before using them.
4817	*
4818	* @remarks No-long-jump zone!!!
4819	*/
4820	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4821	{
4822	int rc = VINF_SUCCESS;
4823	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4824	{
4825	uint64_t u64Val = 0;
4826	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4827	AssertRCReturn(rc, rc);
4828
4829	pMixedCtx->rip = u64Val;
4830	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4831	}
4832	return rc;
4833	}
4834
4835
4836	/**
4837	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4838	*
4839	* @returns VBox status code.
4840	* @param pVCpu Pointer to the VMCPU.
4841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4842	* out-of-sync. Make sure to update the required fields
4843	* before using them.
4844	*
4845	* @remarks No-long-jump zone!!!
4846	*/
4847	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4848	{
4849	int rc = VINF_SUCCESS;
4850	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4851	{
4852	uint64_t u64Val = 0;
4853	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4854	AssertRCReturn(rc, rc);
4855
4856	pMixedCtx->rsp = u64Val;
4857	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4858	}
4859	return rc;
4860	}
4861
4862
4863	/**
4864	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4865	*
4866	* @returns VBox status code.
4867	* @param pVCpu Pointer to the VMCPU.
4868	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4869	* out-of-sync. Make sure to update the required fields
4870	* before using them.
4871	*
4872	* @remarks No-long-jump zone!!!
4873	*/
4874	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4875	{
4876	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4877	{
4878	uint32_t uVal = 0;
4879	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4880	AssertRCReturn(rc, rc);
4881
4882	pMixedCtx->eflags.u32 = uVal;
4883	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4884	{
4885	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4886	Log(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4887
4888	pMixedCtx->eflags.Bits.u1VM = 0;
4889	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4890	}
4891
4892	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4893	}
4894	return VINF_SUCCESS;
4895	}
4896
4897
4898	/**
4899	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4900	* guest-CPU context.
4901	*/
4902	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4903	{
4904	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4905	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4906	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4907	return rc;
4908	}
4909
4910
4911	/**
4912	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4913	* from the guest-state area in the VMCS.
4914	*
4915	* @param pVCpu Pointer to the VMCPU.
4916	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4917	* out-of-sync. Make sure to update the required fields
4918	* before using them.
4919	*
4920	* @remarks No-long-jump zone!!!
4921	*/
4922	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4923	{
4924	uint32_t uIntrState = 0;
4925	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4926	AssertRC(rc);
4927
4928	if (!uIntrState)
4929	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4930	else
4931	{
4932	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4933	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4934	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4935	AssertRC(rc);
4936	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4937	AssertRC(rc);
4938
4939	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4940	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4941	}
4942	}
4943
4944
4945	/**
4946	* Saves the guest's activity state.
4947	*
4948	* @returns VBox status code.
4949	* @param pVCpu Pointer to the VMCPU.
4950	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4951	* out-of-sync. Make sure to update the required fields
4952	* before using them.
4953	*
4954	* @remarks No-long-jump zone!!!
4955	*/
4956	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4957	{
4958	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4959	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4960	return VINF_SUCCESS;
4961	}
4962
4963
4964	/**
4965	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4966	* the current VMCS into the guest-CPU context.
4967	*
4968	* @returns VBox status code.
4969	* @param pVCpu Pointer to the VMCPU.
4970	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4971	* out-of-sync. Make sure to update the required fields
4972	* before using them.
4973	*
4974	* @remarks No-long-jump zone!!!
4975	*/
4976	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4977	{
4978	int rc = VINF_SUCCESS;
4979	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4980	{
4981	uint32_t u32Val = 0;
4982	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4983	pMixedCtx->SysEnter.cs = u32Val;
4984	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4985	}
4986
4987	uint64_t u64Val = 0;
4988	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4989	{
4990	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
4991	pMixedCtx->SysEnter.eip = u64Val;
4992	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4993	}
4994	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4995	{
4996	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
4997	pMixedCtx->SysEnter.esp = u64Val;
4998	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4999	}
5000	return rc;
5001	}
5002
5003
5004	/**
5005	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5006	* context.
5007	*
5008	* @returns VBox status code.
5009	* @param pVCpu Pointer to the VMCPU.
5010	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5011	* out-of-sync. Make sure to update the required fields
5012	* before using them.
5013	*
5014	* @remarks No-long-jump zone!!!
5015	*/
5016	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5017	{
5018	int rc = VINF_SUCCESS;
5019	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5020	{
5021	uint64_t u64Val = 0;
5022	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5023	pMixedCtx->fs.u64Base = u64Val;
5024	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5025	}
5026	return rc;
5027	}
5028
5029
5030	/**
5031	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5032	* context.
5033	*
5034	* @returns VBox status code.
5035	* @param pVCpu Pointer to the VMCPU.
5036	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5037	* out-of-sync. Make sure to update the required fields
5038	* before using them.
5039	*
5040	* @remarks No-long-jump zone!!!
5041	*/
5042	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5043	{
5044	int rc = VINF_SUCCESS;
5045	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5046	{
5047	uint64_t u64Val = 0;
5048	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5049	pMixedCtx->gs.u64Base = u64Val;
5050	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5051	}
5052	return rc;
5053	}
5054
5055
5056	/**
5057	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5058	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5059	* and TSC_AUX.
5060	*
5061	* @returns VBox status code.
5062	* @param pVCpu Pointer to the VMCPU.
5063	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5064	* out-of-sync. Make sure to update the required fields
5065	* before using them.
5066	*
5067	* @remarks No-long-jump zone!!!
5068	*/
5069	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5070	{
5071	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5072	return VINF_SUCCESS;
5073
5074	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5075	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5076	{
5077	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5078	pMsr += i;
5079	switch (pMsr->u32IndexMSR)
5080	{
5081	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5082	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5083	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5084	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5085	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5086	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5087	default:
5088	{
5089	AssertFailed();
5090	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5091	}
5092	}
5093	}
5094	#endif
5095
5096	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5097	return VINF_SUCCESS;
5098	}
5099
5100
5101	/**
5102	* Saves the guest control registers from the current VMCS into the guest-CPU
5103	* context.
5104	*
5105	* @returns VBox status code.
5106	* @param pVCpu Pointer to the VMCPU.
5107	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5108	* out-of-sync. Make sure to update the required fields
5109	* before using them.
5110	*
5111	* @remarks No-long-jump zone!!!
5112	*/
5113	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5114	{
5115	/* Guest CR0. Guest FPU. */
5116	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5117	AssertRCReturn(rc, rc);
5118
5119	/* Guest CR4. */
5120	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5121	AssertRCReturn(rc, rc);
5122
5123	/* Guest CR2 - updated always during the world-switch or in #PF. */
5124	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5125	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5126	{
5127	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5128	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5129
5130	PVM pVM = pVCpu->CTX_SUFF(pVM);
5131	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5132	\|\| ( pVM->hm.s.fNestedPaging
5133	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5134	{
5135	uint64_t u64Val = 0;
5136	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5137	if (pMixedCtx->cr3 != u64Val)
5138	{
5139	CPUMSetGuestCR3(pVCpu, u64Val);
5140	if (VMMRZCallRing3IsEnabled(pVCpu))
5141	{
5142	PGMUpdateCR3(pVCpu, u64Val);
5143	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5144	}
5145	else
5146	{
5147	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5148	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5149	}
5150	}
5151
5152	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5153	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5154	{
5155	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5156	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5157	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5158	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5159
5160	if (VMMRZCallRing3IsEnabled(pVCpu))
5161	{
5162	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5163	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5164	}
5165	else
5166	{
5167	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5168	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5169	}
5170	}
5171	}
5172
5173	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5174	}
5175
5176	/*
5177	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5178	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5179	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5180	*
5181	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5182	*/
5183	if (VMMRZCallRing3IsEnabled(pVCpu))
5184	{
5185	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5186	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5187
5188	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5189	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5190
5191	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5192	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5193	}
5194
5195	return rc;
5196	}
5197
5198
5199	/**
5200	* Reads a guest segment register from the current VMCS into the guest-CPU
5201	* context.
5202	*
5203	* @returns VBox status code.
5204	* @param pVCpu Pointer to the VMCPU.
5205	* @param idxSel Index of the selector in the VMCS.
5206	* @param idxLimit Index of the segment limit in the VMCS.
5207	* @param idxBase Index of the segment base in the VMCS.
5208	* @param idxAccess Index of the access rights of the segment in the VMCS.
5209	* @param pSelReg Pointer to the segment selector.
5210	*
5211	* @remarks No-long-jump zone!!!
5212	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5213	* macro as that takes care of whether to read from the VMCS cache or
5214	* not.
5215	*/
5216	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5217	PCPUMSELREG pSelReg)
5218	{
5219	uint32_t u32Val = 0;
5220	int rc = VMXReadVmcs32(idxSel, &u32Val);
5221	AssertRCReturn(rc, rc);
5222	pSelReg->Sel = (uint16_t)u32Val;
5223	pSelReg->ValidSel = (uint16_t)u32Val;
5224	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5225
5226	rc = VMXReadVmcs32(idxLimit, &u32Val);
5227	AssertRCReturn(rc, rc);
5228	pSelReg->u32Limit = u32Val;
5229
5230	uint64_t u64Val = 0;
5231	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5232	AssertRCReturn(rc, rc);
5233	pSelReg->u64Base = u64Val;
5234
5235	rc = VMXReadVmcs32(idxAccess, &u32Val);
5236	AssertRCReturn(rc, rc);
5237	pSelReg->Attr.u = u32Val;
5238
5239	/*
5240	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5241	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5242	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5243	*
5244	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5245	*/
5246	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5247	{
5248	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5249	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5250	}
5251	return VINF_SUCCESS;
5252	}
5253
5254
5255	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5256	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5257	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5258	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5259	#else
5260	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5261	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5262	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5263	#endif
5264
5265
5266	/**
5267	* Saves the guest segment registers from the current VMCS into the guest-CPU
5268	* context.
5269	*
5270	* @returns VBox status code.
5271	* @param pVCpu Pointer to the VMCPU.
5272	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5273	* out-of-sync. Make sure to update the required fields
5274	* before using them.
5275	*
5276	* @remarks No-long-jump zone!!!
5277	*/
5278	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5279	{
5280	/* Guest segment registers. */
5281	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5282	{
5283	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5284	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5285	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5286	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5287	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5288	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5289	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5290
5291	/* Restore segment attributes for real-on-v86 mode hack. */
5292	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5293	{
5294	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5295	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5296	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5297	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5298	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5299	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5300	}
5301	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5302	}
5303
5304	return VINF_SUCCESS;
5305	}
5306
5307
5308	/**
5309	* Saves the guest descriptor table registers and task register from the current
5310	* VMCS into the guest-CPU context.
5311	*
5312	* @returns VBox status code.
5313	* @param pVCpu Pointer to the VMCPU.
5314	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5315	* out-of-sync. Make sure to update the required fields
5316	* before using them.
5317	*
5318	* @remarks No-long-jump zone!!!
5319	*/
5320	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5321	{
5322	int rc = VINF_SUCCESS;
5323
5324	/* Guest LDTR. */
5325	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5326	{
5327	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5328	AssertRCReturn(rc, rc);
5329	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5330	}
5331
5332	/* Guest GDTR. */
5333	uint64_t u64Val = 0;
5334	uint32_t u32Val = 0;
5335	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5336	{
5337	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5338	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5339	pMixedCtx->gdtr.pGdt = u64Val;
5340	pMixedCtx->gdtr.cbGdt = u32Val;
5341	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5342	}
5343
5344	/* Guest IDTR. */
5345	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5346	{
5347	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5348	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5349	pMixedCtx->idtr.pIdt = u64Val;
5350	pMixedCtx->idtr.cbIdt = u32Val;
5351	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5352	}
5353
5354	/* Guest TR. */
5355	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5356	{
5357	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5358	AssertRCReturn(rc, rc);
5359
5360	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5361	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5362	{
5363	rc = VMXLOCAL_READ_SEG(TR, tr);
5364	AssertRCReturn(rc, rc);
5365	}
5366	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5367	}
5368	return rc;
5369	}
5370
5371	#undef VMXLOCAL_READ_SEG
5372
5373
5374	/**
5375	* Saves the guest debug registers from the current VMCS into the guest-CPU
5376	* context.
5377	*
5378	* @returns VBox status code.
5379	* @param pVCpu Pointer to the VMCPU.
5380	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5381	* out-of-sync. Make sure to update the required fields
5382	* before using them.
5383	*
5384	* @remarks No-long-jump zone!!!
5385	*/
5386	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5387	{
5388	int rc = VINF_SUCCESS;
5389	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5390	{
5391	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5392	uint32_t u32Val;
5393	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5394	pMixedCtx->dr[7] = u32Val;
5395
5396	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5397	}
5398	return rc;
5399	}
5400
5401
5402	/**
5403	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5404	*
5405	* @returns VBox status code.
5406	* @param pVCpu Pointer to the VMCPU.
5407	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5408	* out-of-sync. Make sure to update the required fields
5409	* before using them.
5410	*
5411	* @remarks No-long-jump zone!!!
5412	*/
5413	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5414	{
5415	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5416	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5417	return VINF_SUCCESS;
5418	}
5419
5420
5421	/**
5422	* Saves the entire guest state from the currently active VMCS into the
5423	* guest-CPU context. This essentially VMREADs all guest-data.
5424	*
5425	* @returns VBox status code.
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	*/
5431	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5432	{
5433	Assert(pVCpu);
5434	Assert(pMixedCtx);
5435
5436	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5437	return VINF_SUCCESS;
5438
5439	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5440	there is no real need to. */
5441	if (VMMRZCallRing3IsEnabled(pVCpu))
5442	VMMR0LogFlushDisable(pVCpu);
5443	else
5444	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5445	LogFunc(("\n"));
5446
5447	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5448	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5449
5450	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5451	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5452
5453	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5455
5456	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5458
5459	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5461
5462	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5463	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5464
5465	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5466	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5467
5468	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5469	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5470
5471	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5472	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5473
5474	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5475	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5476
5477	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5478	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5479
5480	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5481	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5482
5483	if (VMMRZCallRing3IsEnabled(pVCpu))
5484	VMMR0LogFlushEnable(pVCpu);
5485
5486	return rc;
5487	}
5488
5489
5490	/**
5491	* Check per-VM and per-VCPU force flag actions that require us to go back to
5492	* ring-3 for one reason or another.
5493	*
5494	* @returns VBox status code (information status code included).
5495	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5496	* ring-3.
5497	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5498	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5499	* interrupts)
5500	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5501	* all EMTs to be in ring-3.
5502	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5503	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5504	* to the EM loop.
5505	*
5506	* @param pVM Pointer to the VM.
5507	* @param pVCpu Pointer to the VMCPU.
5508	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5509	* out-of-sync. Make sure to update the required fields
5510	* before using them.
5511	*/
5512	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5513	{
5514	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5515
5516	int rc = VERR_INTERNAL_ERROR_5;
5517	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5518	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5519	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5520	{
5521	/* We need the control registers now, make sure the guest-CPU context is updated. */
5522	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5523	AssertRCReturn(rc, rc);
5524
5525	/* Pending HM CR3 sync. */
5526	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5527	{
5528	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5529	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5530	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5531	}
5532
5533	/* Pending HM PAE PDPEs. */
5534	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5535	{
5536	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5537	AssertRC(rc);
5538	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5539	}
5540
5541	/* Pending PGM C3 sync. */
5542	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5543	{
5544	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5545	if (rc != VINF_SUCCESS)
5546	{
5547	AssertRC(rc);
5548	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5549	return rc;
5550	}
5551	}
5552
5553	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5554	/* -XXX- what was that about single stepping? */
5555	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5556	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5557	{
5558	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5559	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5560	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5561	return rc;
5562	}
5563
5564	/* Pending VM request packets, such as hardware interrupts. */
5565	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5566	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5567	{
5568	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5569	return VINF_EM_PENDING_REQUEST;
5570	}
5571
5572	/* Pending PGM pool flushes. */
5573	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5574	{
5575	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5576	return VINF_PGM_POOL_FLUSH_PENDING;
5577	}
5578
5579	/* Pending DMA requests. */
5580	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5581	{
5582	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5583	return VINF_EM_RAW_TO_R3;
5584	}
5585	}
5586
5587	/* Paranoia. */
5588	Assert(rc != VERR_EM_INTERPRETER);
5589	return VINF_SUCCESS;
5590	}
5591
5592
5593	/**
5594	* Converts any TRPM trap into a pending VMX event. This is typically used when
5595	* entering from ring-3 (not longjmp returns).
5596	*
5597	* @param pVCpu Pointer to the VMCPU.
5598	*/
5599	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5600	{
5601	Assert(TRPMHasTrap(pVCpu));
5602	Assert(!pVCpu->hm.s.Event.fPending);
5603
5604	uint8_t uVector;
5605	TRPMEVENT enmTrpmEvent;
5606	RTGCUINT uErrCode;
5607	RTGCUINTPTR GCPtrFaultAddress;
5608	uint8_t cbInstr;
5609
5610	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5611	AssertRC(rc);
5612
5613	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5614	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5615	if (enmTrpmEvent == TRPM_TRAP)
5616	{
5617	switch (uVector)
5618	{
5619	case X86_XCPT_BP:
5620	case X86_XCPT_OF:
5621	{
5622	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5623	break;
5624	}
5625
5626	case X86_XCPT_PF:
5627	case X86_XCPT_DF:
5628	case X86_XCPT_TS:
5629	case X86_XCPT_NP:
5630	case X86_XCPT_SS:
5631	case X86_XCPT_GP:
5632	case X86_XCPT_AC:
5633	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5634	/* no break! */
5635	default:
5636	{
5637	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5638	break;
5639	}
5640	}
5641	}
5642	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5643	{
5644	if (uVector != X86_XCPT_NMI)
5645	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5646	else
5647	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5648	}
5649	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5650	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5651	else
5652	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5653
5654	rc = TRPMResetTrap(pVCpu);
5655	AssertRC(rc);
5656	Log(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5657	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5658	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5659	}
5660
5661
5662	/**
5663	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5664	* VT-x to execute any instruction.
5665	*
5666	* @param pvCpu Pointer to the VMCPU.
5667	*/
5668	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5669	{
5670	Assert(pVCpu->hm.s.Event.fPending);
5671
5672	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5673	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5674	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5675	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5676
5677	/* If a trap was already pending, we did something wrong! */
5678	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5679
5680	TRPMEVENT enmTrapType;
5681	switch (uVectorType)
5682	{
5683	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5684	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5685	enmTrapType = TRPM_HARDWARE_INT;
5686	break;
5687	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5688	enmTrapType = TRPM_SOFTWARE_INT;
5689	break;
5690	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5691	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5692	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5693	enmTrapType = TRPM_TRAP;
5694	break;
5695	default:
5696	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5697	enmTrapType = TRPM_32BIT_HACK;
5698	break;
5699	}
5700
5701	Log(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5702
5703	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5704	AssertRC(rc);
5705
5706	if (fErrorCodeValid)
5707	TRPMSetErrorCode(pVCpu, uErrorCode);
5708	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5709	&& uVector == X86_XCPT_PF)
5710	{
5711	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5712	}
5713	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5714	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5715	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5716	{
5717	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5718	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5719	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5720	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5721	}
5722	pVCpu->hm.s.Event.fPending = false;
5723	}
5724
5725
5726	/**
5727	* Does the necessary state syncing before doing a longjmp to ring-3.
5728	*
5729	* @param pVM Pointer to the VM.
5730	* @param pVCpu Pointer to the VMCPU.
5731	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5732	* out-of-sync. Make sure to update the required fields
5733	* before using them.
5734	* @param rcExit The reason for exiting to ring-3. Can be
5735	* VINF_VMM_UNKNOWN_RING3_CALL.
5736	*
5737	* @remarks No-long-jmp zone!!!
5738	*/
5739	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5740	{
5741	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5742	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5743
5744	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5745	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5746	AssertRC(rc);
5747
5748	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5749	if (CPUMIsGuestFPUStateActive(pVCpu))
5750	{
5751	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5752	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5753	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5754	}
5755
5756	/* Restore debug registers if necessary and resync on next R0 reentry. */
5757	if (CPUMIsGuestDebugStateActive(pVCpu))
5758	{
5759	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5760	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5761	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5762	}
5763	else if (CPUMIsHyperDebugStateActive(pVCpu))
5764	{
5765	CPUMR0LoadHostDebugState(pVM, pVCpu);
5766	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5767	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5768	}
5769
5770	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5771	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5772	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5773	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5774	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5775	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5776	}
5777
5778
5779	/**
5780	* An action requires us to go back to ring-3. This function does the necessary
5781	* steps before we can safely return to ring-3. This is not the same as longjmps
5782	* to ring-3, this is voluntary.
5783	*
5784	* @param pVM Pointer to the VM.
5785	* @param pVCpu Pointer to the VMCPU.
5786	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5787	* out-of-sync. Make sure to update the required fields
5788	* before using them.
5789	* @param rcExit The reason for exiting to ring-3. Can be
5790	* VINF_VMM_UNKNOWN_RING3_CALL.
5791	*/
5792	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5793	{
5794	Assert(pVM);
5795	Assert(pVCpu);
5796	Assert(pMixedCtx);
5797	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5798
5799	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5800	{
5801	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5802	return;
5803	}
5804	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5805	{
5806	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5807	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5808	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5809	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5810	return;
5811	}
5812
5813	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5814	VMMRZCallRing3Disable(pVCpu);
5815	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5816
5817	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5818	if (pVCpu->hm.s.Event.fPending)
5819	{
5820	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5821	Assert(!pVCpu->hm.s.Event.fPending);
5822	}
5823
5824	/* Sync. the guest state. */
5825	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5826	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5827
5828	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5829	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5830	\| CPUM_CHANGED_LDTR
5831	\| CPUM_CHANGED_GDTR
5832	\| CPUM_CHANGED_IDTR
5833	\| CPUM_CHANGED_TR
5834	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5835
5836	/* On our way back from ring-3 the following needs to be done. */
5837	/** @todo This can change with preemption hooks. */
5838	if (rcExit == VINF_EM_RAW_INTERRUPT)
5839	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5840	else
5841	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5842
5843	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5844	VMMRZCallRing3Enable(pVCpu);
5845	}
5846
5847
5848	/**
5849	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5850	* longjump to ring-3 and possibly get preempted.
5851	*
5852	* @param pVCpu Pointer to the VMCPU.
5853	* @param enmOperation The operation causing the ring-3 longjump.
5854	* @param pvUser The user argument (pointer to the possibly
5855	* out-of-date guest-CPU context).
5856	*
5857	* @remarks Must never be called with @a enmOperation ==
5858	* VMMCALLRING3_VM_R0_ASSERTION.
5859	*/
5860	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5861	{
5862	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5863	Assert(pVCpu);
5864	Assert(pvUser);
5865	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5866	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5867
5868	VMMRZCallRing3Disable(pVCpu);
5869	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5870	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5871	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5872	VMMRZCallRing3Enable(pVCpu);
5873	}
5874
5875
5876	/**
5877	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5878	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5879	*
5880	* @returns VBox status code.
5881	* @param pVCpu Pointer to the VMCPU.
5882	*/
5883	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5884	{
5885	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5886	{
5887	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5888	{
5889	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
5890	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5891	AssertRC(rc);
5892	}
5893	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5894	}
5895
5896
5897	/**
5898	* Injects any pending events into the guest if the guest is in a state to
5899	* receive them.
5900	*
5901	* @returns VBox status code (informational status codes included).
5902	* @param pVCpu Pointer to the VMCPU.
5903	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5904	* out-of-sync. Make sure to update the required fields
5905	* before using them.
5906	*/
5907	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5908	{
5909	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5910	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5911	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5912	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5913
5914	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5915	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5916	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5917	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5918	Assert(!TRPMHasTrap(pVCpu));
5919
5920	int rc = VINF_SUCCESS;
5921	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5922	{
5923	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5924	bool fInject = true;
5925	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5926	{
5927	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5928	AssertRCReturn(rc, rc);
5929	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5930	if ( fBlockInt
5931	\|\| fBlockSti
5932	\|\| fBlockMovSS)
5933	{
5934	fInject = false;
5935	}
5936	}
5937	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5938	&& ( fBlockMovSS
5939	\|\| fBlockSti))
5940	{
5941	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5942	fInject = false;
5943	}
5944
5945	if (fInject)
5946	{
5947	Log(("Injecting pending event\n"));
5948	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5949	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5950	AssertRCReturn(rc, rc);
5951	pVCpu->hm.s.Event.fPending = false;
5952	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5953	}
5954	else
5955	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5956	} /** @todo SMI. SMIs take priority over NMIs. */
5957	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5958	{
5959	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5960	if ( !fBlockMovSS
5961	&& !fBlockSti)
5962	{
5963	Log(("Injecting NMI\n"));
5964	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5965	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5966	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5967	0 /* GCPtrFaultAddress */, &uIntrState);
5968	AssertRCReturn(rc, rc);
5969	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5970	}
5971	else
5972	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5973	}
5974	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5975	{
5976	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5977	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5978	AssertRCReturn(rc, rc);
5979	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5980	if ( !fBlockInt
5981	&& !fBlockSti
5982	&& !fBlockMovSS)
5983	{
5984	uint8_t u8Interrupt;
5985	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5986	if (RT_SUCCESS(rc))
5987	{
5988	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5989	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5990	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5991	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5992	0 /* GCPtrFaultAddress */, &uIntrState);
5993	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5994	}
5995	else
5996	{
5997	/** @todo Does this actually happen? If not turn it into an assertion. */
5998	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5999	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6000	rc = VINF_SUCCESS;
6001	}
6002	}
6003	else
6004	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6005	}
6006
6007	/*
6008	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6009	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6010	*/
6011	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6012	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6013	int rc2 = VINF_SUCCESS;
6014	if ( fBlockSti
6015	\|\| fBlockMovSS)
6016	{
6017	if (!DBGFIsStepping(pVCpu))
6018	{
6019	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6020	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6021	{
6022	/*
6023	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6024	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6025	*/
6026	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6027	AssertRCReturn(rc, rc);
6028	}
6029	}
6030	else
6031	{
6032	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6033	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6034	uIntrState = 0;
6035	}
6036	}
6037
6038	/*
6039	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6040	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6041	*/
6042	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6043	AssertRC(rc2);
6044
6045	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6046	return rc;
6047	}
6048
6049
6050	/**
6051	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6052	*
6053	* @param pVCpu Pointer to the VMCPU.
6054	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6055	* out-of-sync. Make sure to update the required fields
6056	* before using them.
6057	*/
6058	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6059	{
6060	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6061	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6062	}
6063
6064
6065	/**
6066	* Injects a double-fault (#DF) exception into the VM.
6067	*
6068	* @returns VBox status code (informational status code included).
6069	* @param pVCpu Pointer to the VMCPU.
6070	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6071	* out-of-sync. Make sure to update the required fields
6072	* before using them.
6073	*/
6074	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6075	{
6076	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6077	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6078	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6079	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6080	puIntrState);
6081	}
6082
6083
6084	/**
6085	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6086	*
6087	* @param pVCpu Pointer to the VMCPU.
6088	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6089	* out-of-sync. Make sure to update the required fields
6090	* before using them.
6091	*/
6092	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6093	{
6094	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6095	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6096	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6097	}
6098
6099
6100	/**
6101	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6102	*
6103	* @param pVCpu Pointer to the VMCPU.
6104	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6105	* out-of-sync. Make sure to update the required fields
6106	* before using them.
6107	* @param cbInstr The value of RIP that is to be pushed on the guest
6108	* stack.
6109	*/
6110	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6111	{
6112	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6113	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6114	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6115	}
6116
6117
6118	/**
6119	* Injects a general-protection (#GP) fault into the VM.
6120	*
6121	* @returns VBox status code (informational status code included).
6122	* @param pVCpu Pointer to the VMCPU.
6123	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6124	* out-of-sync. Make sure to update the required fields
6125	* before using them.
6126	* @param u32ErrorCode The error code associated with the #GP.
6127	*/
6128	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6129	uint32_t *puIntrState)
6130	{
6131	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6132	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6133	if (fErrorCodeValid)
6134	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6135	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6136	puIntrState);
6137	}
6138
6139
6140	/**
6141	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6142	*
6143	* @param pVCpu Pointer to the VMCPU.
6144	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6145	* out-of-sync. Make sure to update the required fields
6146	* before using them.
6147	* @param uVector The software interrupt vector number.
6148	* @param cbInstr The value of RIP that is to be pushed on the guest
6149	* stack.
6150	*/
6151	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6152	{
6153	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6154	if ( uVector == X86_XCPT_BP
6155	\|\| uVector == X86_XCPT_OF)
6156	{
6157	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6158	}
6159	else
6160	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6161	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6162	}
6163
6164
6165	/**
6166	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6167	* stack.
6168	*
6169	* @returns VBox status code (information status code included).
6170	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6171	* @param pVM Pointer to the VM.
6172	* @param pMixedCtx Pointer to the guest-CPU context.
6173	* @param uValue The value to push to the guest stack.
6174	*/
6175	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6176	{
6177	/*
6178	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6179	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6180	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6181	*/
6182	if (pMixedCtx->sp == 1)
6183	return VINF_EM_RESET;
6184	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6185	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6186	AssertRCReturn(rc, rc);
6187	return rc;
6188	}
6189
6190
6191	/**
6192	* Injects an event into the guest upon VM-entry by updating the relevant fields
6193	* in the VM-entry area in the VMCS.
6194	*
6195	* @returns VBox status code (informational error codes included).
6196	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6197	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6198	*
6199	* @param pVCpu Pointer to the VMCPU.
6200	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6201	* be out-of-sync. Make sure to update the required
6202	* fields before using them.
6203	* @param u64IntrInfo The VM-entry interruption-information field.
6204	* @param cbInstr The VM-entry instruction length in bytes (for
6205	* software interrupts, exceptions and privileged
6206	* software exceptions).
6207	* @param u32ErrCode The VM-entry exception error code.
6208	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6209	* @param puIntrState Pointer to the current guest interruptibility-state.
6210	* This interruptibility-state will be updated if
6211	* necessary. This cannot not be NULL.
6212	*
6213	* @remarks No-long-jump zone!!!
6214	* @remarks Requires CR0!
6215	*/
6216	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6217	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6218	{
6219	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6220	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6221	Assert(puIntrState);
6222	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6223
6224	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6225	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6226
6227	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6228	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6229	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6230
6231	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6232
6233	/* We require CR0 to check if the guest is in real-mode. */
6234	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6235	AssertRCReturn(rc, rc);
6236
6237	/*
6238	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6239	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6240	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6241	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6242	*/
6243	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6244	{
6245	PVM pVM = pVCpu->CTX_SUFF(pVM);
6246	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6247	{
6248	Assert(PDMVmmDevHeapIsEnabled(pVM));
6249	Assert(pVM->hm.s.vmx.pRealModeTSS);
6250
6251	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6252	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6253	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6254	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6255	AssertRCReturn(rc, rc);
6256	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6257
6258	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6259	const size_t cbIdtEntry = 4;
6260	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6261	{
6262	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6263	if (uVector == X86_XCPT_DF)
6264	return VINF_EM_RESET;
6265	else if (uVector == X86_XCPT_GP)
6266	{
6267	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6268	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6269	}
6270
6271	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6272	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6273	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6274	}
6275
6276	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6277	uint16_t uGuestIp = pMixedCtx->ip;
6278	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6279	{
6280	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6281	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6282	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6283	}
6284	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6285	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6286
6287	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6288	uint16_t offIdtEntry = 0;
6289	RTSEL selIdtEntry = 0;
6290	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6291	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6292	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6293	AssertRCReturn(rc, rc);
6294
6295	/* Construct the stack frame for the interrupt/exception handler. */
6296	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6297	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6298	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6299	AssertRCReturn(rc, rc);
6300
6301	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6302	if (rc == VINF_SUCCESS)
6303	{
6304	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6305	pMixedCtx->rip = offIdtEntry;
6306	pMixedCtx->cs.Sel = selIdtEntry;
6307	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6308	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6309	&& uVector == X86_XCPT_PF)
6310	{
6311	pMixedCtx->cr2 = GCPtrFaultAddress;
6312	}
6313	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6314	\| HM_CHANGED_GUEST_RIP
6315	\| HM_CHANGED_GUEST_RFLAGS
6316	\| HM_CHANGED_GUEST_RSP;
6317
6318	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6319	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6320	{
6321	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6322	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6323	Log(("Clearing inhibition due to STI.\n"));
6324	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6325	}
6326	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6327	}
6328	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6329	return rc;
6330	}
6331	else
6332	{
6333	/*
6334	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6335	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6336	*/
6337	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6338	}
6339	}
6340
6341	/* Validate. */
6342	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6343	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6344	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6345
6346	/* Inject. */
6347	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6348	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6349	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6350	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6351
6352	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6353	&& uVector == X86_XCPT_PF)
6354	{
6355	pMixedCtx->cr2 = GCPtrFaultAddress;
6356	}
6357	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6358
6359	AssertRCReturn(rc, rc);
6360	return rc;
6361	}
6362
6363
6364	/**
6365	* Enters the VT-x session.
6366	*
6367	* @returns VBox status code.
6368	* @param pVM Pointer to the VM.
6369	* @param pVCpu Pointer to the VMCPU.
6370	* @param pCpu Pointer to the CPU info struct.
6371	*/
6372	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6373	{
6374	AssertPtr(pVM);
6375	AssertPtr(pVCpu);
6376	Assert(pVM->hm.s.vmx.fSupported);
6377	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6378	NOREF(pCpu);
6379
6380	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6381
6382	/* Make sure we're in VMX root mode. */
6383	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6384	if (!(u32HostCR4 & X86_CR4_VMXE))
6385	{
6386	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6387	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6388	}
6389
6390	/* Load the active VMCS as the current one. */
6391	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6392	if (RT_FAILURE(rc))
6393	return rc;
6394
6395	/** @todo this will change with preemption hooks where can can VMRESUME as long
6396	* as we're no preempted. */
6397	pVCpu->hm.s.fResumeVM = false;
6398	return VINF_SUCCESS;
6399	}
6400
6401
6402	/**
6403	* Leaves the VT-x session.
6404	*
6405	* @returns VBox status code.
6406	* @param pVM Pointer to the VM.
6407	* @param pVCpu Pointer to the VMCPU.
6408	* @param pCtx Pointer to the guest-CPU context.
6409	*/
6410	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6411	{
6412	AssertPtr(pVCpu);
6413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6414	NOREF(pVM);
6415	NOREF(pCtx);
6416
6417	/** @todo this will change with preemption hooks where we only VMCLEAR when
6418	* we are actually going to be preempted, not all the time like we
6419	* currently do. */
6420	/*
6421	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6422	* and mark the VMCS launch-state as "clear".
6423	*/
6424	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6425	return rc;
6426	}
6427
6428
6429	/**
6430	* Saves the host state in the VMCS host-state.
6431	* Sets up the VM-exit MSR-load area.
6432	*
6433	* The CPU state will be loaded from these fields on every successful VM-exit.
6434	*
6435	* @returns VBox status code.
6436	* @param pVM Pointer to the VM.
6437	* @param pVCpu Pointer to the VMCPU.
6438	*
6439	* @remarks No-long-jump zone!!!
6440	*/
6441	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6442	{
6443	AssertPtr(pVM);
6444	AssertPtr(pVCpu);
6445	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6446
6447	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6448
6449	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6450	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6451	return VINF_SUCCESS;
6452
6453	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6455
6456	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6458
6459	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6461
6462	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6463	return rc;
6464	}
6465
6466
6467	/**
6468	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6469	* loaded from these fields on every successful VM-entry.
6470	*
6471	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6472	* Sets up the VM-entry controls.
6473	* Sets up the appropriate VMX non-root function to execute guest code based on
6474	* the guest CPU mode.
6475	*
6476	* @returns VBox status code.
6477	* @param pVM Pointer to the VM.
6478	* @param pVCpu Pointer to the VMCPU.
6479	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6480	* out-of-sync. Make sure to update the required fields
6481	* before using them.
6482	*
6483	* @remarks No-long-jump zone!!!
6484	*/
6485	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6486	{
6487	AssertPtr(pVM);
6488	AssertPtr(pVCpu);
6489	AssertPtr(pMixedCtx);
6490	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6491
6492	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6493
6494	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6495
6496	/* Determine real-on-v86 mode. */
6497	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6498	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6499	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6500	{
6501	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6502	}
6503
6504	/*
6505	* Load the guest-state into the VMCS.
6506	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6507	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6508	*/
6509	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6510	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6511
6512	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6513	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6514
6515	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6516	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6517
6518	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6519	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6520
6521	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6522	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6523	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6524
6525	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6526	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6527
6528	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6529	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6530
6531	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6532	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6533
6534	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6535	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6536
6537	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6538	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6539
6540	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6541	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6542	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6543
6544	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6545	return rc;
6546	}
6547
6548
6549	/**
6550	* Does the preparations before executing guest code in VT-x.
6551	*
6552	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6553	* recompiler. We must be cautious what we do here regarding committing
6554	* guest-state information into the the VMCS assuming we assuredly execute the
6555	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6556	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6557	* that the recompiler can (and should) use them when it resumes guest
6558	* execution. Otherwise such operations must be done when we can no longer
6559	* exit to ring-3.
6560	*
6561	* @returns VBox status code (informational status codes included).
6562	* @retval VINF_SUCCESS if we can proceed with running the guest.
6563	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6564	* into the guest.
6565	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6566	*
6567	* @param pVM Pointer to the VM.
6568	* @param pVCpu Pointer to the VMCPU.
6569	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6570	* out-of-sync. Make sure to update the required fields
6571	* before using them.
6572	* @param pVmxTransient Pointer to the VMX transient structure.
6573	*
6574	* @remarks Called with preemption disabled.
6575	*/
6576	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6577	{
6578	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6579
6580	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6581	PGMRZDynMapFlushAutoSet(pVCpu);
6582	#endif
6583
6584	/* Check force flag actions that might require us to go back to ring-3. */
6585	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6586	if (rc != VINF_SUCCESS)
6587	return rc;
6588
6589	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6590	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6591	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6592	{
6593	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6594	RTGCPHYS GCPhysApicBase;
6595	GCPhysApicBase = pMixedCtx->msrApicBase;
6596	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6597
6598	/* Unalias any existing mapping. */
6599	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6600	AssertRCReturn(rc, rc);
6601
6602	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6603	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6604	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6605	AssertRCReturn(rc, rc);
6606
6607	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6608	}
6609
6610	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6611	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6612	pVmxTransient->uEFlags = ASMIntDisableFlags();
6613	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6614	{
6615	ASMSetFlags(pVmxTransient->uEFlags);
6616	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6617	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6618	return VINF_EM_RAW_INTERRUPT;
6619	}
6620	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6621	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6622	#endif
6623
6624	/*
6625	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6626	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6627	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6628	*/
6629	/** @todo Rework event evaluation and injection to be completely separate. */
6630	if (TRPMHasTrap(pVCpu))
6631	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6632
6633	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6634	AssertRCReturn(rc, rc);
6635	return rc;
6636	}
6637
6638
6639	/**
6640	* Prepares to run guest code in VT-x and we've committed to doing so. This
6641	* means there is no backing out to ring-3 or anywhere else at this
6642	* point.
6643	*
6644	* @param pVM Pointer to the VM.
6645	* @param pVCpu Pointer to the VMCPU.
6646	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6647	* out-of-sync. Make sure to update the required fields
6648	* before using them.
6649	* @param pVmxTransient Pointer to the VMX transient structure.
6650	*
6651	* @remarks Called with preemption disabled.
6652	* @remarks No-long-jump zone!!!
6653	*/
6654	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6655	{
6656	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6657	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6658
6659	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6660	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6661	pVmxTransient->uEFlags = ASMIntDisableFlags();
6662	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6663	#endif
6664
6665	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6666	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6667	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6668	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6669	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6670	#endif
6671	int rc = VINF_SUCCESS;
6672	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6673	{
6674	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6675	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6676	}
6677	else if (pVCpu->hm.s.fContextUseFlags)
6678	{
6679	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6680	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6681	}
6682	AssertRC(rc);
6683	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6684
6685	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6686	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6687	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6688
6689	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6690	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6691	{
6692	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6693	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6694	}
6695
6696	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6697	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6698	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6699
6700	/*
6701	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6702	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6703	*/
6704	if (pVM->hm.s.fTPRPatchingActive)
6705	{
6706	Assert(!CPUMIsGuestInLongMode(pVCpu));
6707
6708	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6709	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6710	AssertRC(rc);
6711
6712	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6713	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6714	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6715	see hmR0VmxLoadGuestApicState(). */
6716	}
6717
6718	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6719	/*
6720	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6721	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6722	*/
6723	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6724	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6725	{
6726	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6727	uint64_t u64HostTscAux = 0;
6728	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6729	AssertRC(rc2);
6730	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6731	}
6732	#endif
6733
6734	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6735	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6736	to start executing. */
6737	}
6738
6739
6740	/**
6741	* Performs some essential restoration of state after running guest code in
6742	* VT-x.
6743	*
6744	* @param pVM Pointer to the VM.
6745	* @param pVCpu Pointer to the VMCPU.
6746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6747	* out-of-sync. Make sure to update the required fields
6748	* before using them.
6749	* @param pVmxTransient Pointer to the VMX transient structure.
6750	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6751	*
6752	* @remarks Called with interrupts disabled.
6753	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6754	* unconditionally when it is safe to do so.
6755	*/
6756	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6757	{
6758	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6759	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6760
6761	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6762	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6763	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6764	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6765	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6766
6767	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6768	{
6769	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6770	/* Restore host's TSC_AUX. */
6771	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6772	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6773	#endif
6774	/** @todo Find a way to fix hardcoding a guestimate. */
6775	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6776	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6777	}
6778
6779	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6780	Assert(!(ASMGetFlags() & X86_EFL_IF));
6781	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6782
6783	/* Restore the effects of TPR patching if any. */
6784	if (pVM->hm.s.fTPRPatchingActive)
6785	{
6786	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6787	AssertRC(rc);
6788	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6789	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6790	}
6791
6792	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6793	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6794
6795	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6796	uint32_t uExitReason;
6797	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6798	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6799	AssertRC(rc);
6800	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6801	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6802
6803	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6804	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6805
6806	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6807	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6808	{
6809	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6810	return;
6811	}
6812
6813	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6814	{
6815	/* Update the guest interruptibility-state from the VMCS. */
6816	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6817	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6818	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6819	AssertRC(rc);
6820	#endif
6821	/*
6822	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6823	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6824	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6825	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6826	*/
6827	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6828	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6829	{
6830	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6831	AssertRC(rc);
6832	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6833	}
6834	}
6835	}
6836
6837
6838	/**
6839	* Runs the guest code using VT-x.
6840	*
6841	* @returns VBox status code.
6842	* @param pVM Pointer to the VM.
6843	* @param pVCpu Pointer to the VMCPU.
6844	* @param pCtx Pointer to the guest-CPU context.
6845	*
6846	* @remarks Called with preemption disabled.
6847	*/
6848	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6849	{
6850	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6851	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6852
6853	VMXTRANSIENT VmxTransient;
6854	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6855	int rc = VERR_INTERNAL_ERROR_5;
6856	uint32_t cLoops = 0;
6857
6858	for (;; cLoops++)
6859	{
6860	Assert(!HMR0SuspendPending());
6861	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6862	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6863	(unsigned)RTMpCpuId(), cLoops));
6864
6865	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6866	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6867	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6868	if (rc != VINF_SUCCESS)
6869	break;
6870
6871	/*
6872	* No longjmps to ring-3 from this point on!!!
6873	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6874	* This also disables flushing of the R0-logger instance (if any).
6875	*/
6876	VMMRZCallRing3Disable(pVCpu);
6877	VMMRZCallRing3RemoveNotification(pVCpu);
6878	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6879
6880	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6881	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6882
6883	/*
6884	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6885	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6886	*/
6887	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6888	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6889	{
6890	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6891	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6892	return rc;
6893	}
6894
6895	/* Handle the VM-exit. */
6896	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6897	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6898	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6899	HMVMX_START_EXIT_DISPATCH_PROF();
6900	#ifdef HMVMX_USE_FUNCTION_TABLE
6901	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6902	#else
6903	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6904	#endif
6905	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6906	if (rc != VINF_SUCCESS)
6907	break;
6908	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6909	{
6910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6911	rc = VINF_EM_RAW_INTERRUPT;
6912	break;
6913	}
6914	}
6915
6916	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6917	if (rc == VERR_EM_INTERPRETER)
6918	rc = VINF_EM_RAW_EMULATE_INSTR;
6919	else if (rc == VINF_EM_RESET)
6920	rc = VINF_EM_TRIPLE_FAULT;
6921	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6922	return rc;
6923	}
6924
6925
6926	#ifndef HMVMX_USE_FUNCTION_TABLE
6927	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6928	{
6929	int rc;
6930	switch (rcReason)
6931	{
6932	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6933	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6934	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6935	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6936	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6937	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6938	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6939	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6940	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6941	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6942	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6943	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6944	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6945	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6946	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6947	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6948	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6949	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6950	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6951	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6952	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6953	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6954	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6955	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6956	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6957	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6958	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6959	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6960	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6961	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6962	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6963	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6964	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6965
6966	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6967	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6968	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6969	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6970	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6971	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6972	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6973	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6974	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6975
6976	case VMX_EXIT_VMCALL:
6977	case VMX_EXIT_VMCLEAR:
6978	case VMX_EXIT_VMLAUNCH:
6979	case VMX_EXIT_VMPTRLD:
6980	case VMX_EXIT_VMPTRST:
6981	case VMX_EXIT_VMREAD:
6982	case VMX_EXIT_VMRESUME:
6983	case VMX_EXIT_VMWRITE:
6984	case VMX_EXIT_VMXOFF:
6985	case VMX_EXIT_VMXON:
6986	case VMX_EXIT_INVEPT:
6987	case VMX_EXIT_INVVPID:
6988	case VMX_EXIT_VMFUNC:
6989	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6990	break;
6991	default:
6992	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6993	break;
6994	}
6995	return rc;
6996	}
6997	#endif
6998
6999	#ifdef DEBUG
7000	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7001	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7002	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7003	# define VMX_ASSERT_PREEMPT_CPUID() \
7004	do \
7005	{ \
7006	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7007	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7008	} while (0)
7009
7010	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7011	do { \
7012	AssertPtr(pVCpu); \
7013	AssertPtr(pMixedCtx); \
7014	AssertPtr(pVmxTransient); \
7015	Assert(pVmxTransient->fVMEntryFailed == false); \
7016	Assert(ASMIntAreEnabled()); \
7017	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7018	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7019	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7020	(unsigned)pVCpu->idCpu)); \
7021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7022	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7023	VMX_ASSERT_PREEMPT_CPUID(); \
7024	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7025	} while (0)
7026	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7027	do { \
7028	LogFunc(("\n")); \
7029	} while(0)
7030	#else /* Release builds */
7031	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7032	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7033	#endif
7034
7035
7036	/**
7037	* Advances the guest RIP after reading it from the VMCS.
7038	*
7039	* @returns VBox status code.
7040	* @param pVCpu Pointer to the VMCPU.
7041	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7042	* out-of-sync. Make sure to update the required fields
7043	* before using them.
7044	* @param pVmxTransient Pointer to the VMX transient structure.
7045	*
7046	* @remarks No-long-jump zone!!!
7047	*/
7048	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7049	{
7050	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7051	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7052	AssertRCReturn(rc, rc);
7053
7054	pMixedCtx->rip += pVmxTransient->cbInstr;
7055	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7056	return rc;
7057	}
7058
7059
7060	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7061	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7062	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7063	/**
7064	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7065	*/
7066	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7067	{
7068	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7070	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7071	Assert(ASMIntAreEnabled());
7072	return VINF_SUCCESS;
7073	#else
7074	return VINF_EM_RAW_INTERRUPT;
7075	#endif
7076	}
7077
7078
7079	/**
7080	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7081	*/
7082	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7083	{
7084	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7085	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7086
7087	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7088	AssertRCReturn(rc, rc);
7089
7090	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7091	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7092	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7093
7094	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7095	{
7096	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7097	return VINF_EM_RAW_INTERRUPT;
7098	}
7099
7100	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7101	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7102	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7103	{
7104	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7105	return VINF_SUCCESS;
7106	}
7107	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7108	{
7109	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7110	return rc;
7111	}
7112
7113	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7114	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7115	switch (uIntrType)
7116	{
7117	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7118	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7119	/* no break */
7120	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7121	{
7122	switch (uVector)
7123	{
7124	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7125	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7126	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7127	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7128	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7129	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7130	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7131	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7132	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7133	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7134	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7135	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7136	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7137	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7138	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7139	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7140	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7141	#endif
7142	default:
7143	{
7144	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7145	AssertRCReturn(rc, rc);
7146
7147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7148	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7149	{
7150	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7151	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7152	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7153	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7154	AssertRCReturn(rc, rc);
7155	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7156	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7157	0 /* GCPtrFaultAddress */);
7158	AssertRCReturn(rc, rc);
7159	}
7160	else
7161	{
7162	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7163	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7164	}
7165	break;
7166	}
7167	}
7168	break;
7169	}
7170
7171	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7172	default:
7173	{
7174	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7175	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7176	break;
7177	}
7178	}
7179	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7180	return rc;
7181	}
7182
7183
7184	/**
7185	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7186	*/
7187	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7188	{
7189	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7190
7191	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7192	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7193	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7194	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7195	AssertRCReturn(rc, rc);
7196
7197	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7198	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7199	return VINF_SUCCESS;
7200	}
7201
7202
7203	/**
7204	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7205	*/
7206	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7207	{
7208	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7209	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7210	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7211	}
7212
7213
7214	/**
7215	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7216	*/
7217	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7218	{
7219	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7221	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7222	}
7223
7224
7225	/**
7226	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7227	*/
7228	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7229	{
7230	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7232	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7233	}
7234
7235
7236	/**
7237	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7238	*/
7239	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7240	{
7241	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7242	PVM pVM = pVCpu->CTX_SUFF(pVM);
7243	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7244	if (RT_LIKELY(rc == VINF_SUCCESS))
7245	{
7246	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7247	Assert(pVmxTransient->cbInstr == 2);
7248	}
7249	else
7250	{
7251	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7252	rc = VERR_EM_INTERPRETER;
7253	}
7254	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7255	return rc;
7256	}
7257
7258
7259	/**
7260	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7261	*/
7262	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7263	{
7264	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7265	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7266	AssertRCReturn(rc, rc);
7267
7268	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7269	return VINF_EM_RAW_EMULATE_INSTR;
7270
7271	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7272	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7273	}
7274
7275
7276	/**
7277	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7278	*/
7279	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7280	{
7281	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7282	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7283	AssertRCReturn(rc, rc);
7284
7285	PVM pVM = pVCpu->CTX_SUFF(pVM);
7286	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7287	if (RT_LIKELY(rc == VINF_SUCCESS))
7288	{
7289	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7290	Assert(pVmxTransient->cbInstr == 2);
7291	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7292	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7293	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7294	}
7295	else
7296	{
7297	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7298	rc = VERR_EM_INTERPRETER;
7299	}
7300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7301	return rc;
7302	}
7303
7304
7305	/**
7306	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7307	*/
7308	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7309	{
7310	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7311	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7312	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7313	AssertRCReturn(rc, rc);
7314
7315	PVM pVM = pVCpu->CTX_SUFF(pVM);
7316	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7317	if (RT_LIKELY(rc == VINF_SUCCESS))
7318	{
7319	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7320	Assert(pVmxTransient->cbInstr == 3);
7321	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7322	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7323	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7324	}
7325	else
7326	{
7327	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7328	rc = VERR_EM_INTERPRETER;
7329	}
7330	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7331	return rc;
7332	}
7333
7334
7335	/**
7336	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7337	*/
7338	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7339	{
7340	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7341	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7342	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7343	AssertRCReturn(rc, rc);
7344
7345	PVM pVM = pVCpu->CTX_SUFF(pVM);
7346	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7347	if (RT_LIKELY(rc == VINF_SUCCESS))
7348	{
7349	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7350	Assert(pVmxTransient->cbInstr == 2);
7351	}
7352	else
7353	{
7354	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7355	rc = VERR_EM_INTERPRETER;
7356	}
7357	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7358	return rc;
7359	}
7360
7361
7362	/**
7363	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7364	*/
7365	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7366	{
7367	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7368	PVM pVM = pVCpu->CTX_SUFF(pVM);
7369	Assert(!pVM->hm.s.fNestedPaging);
7370
7371	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7372	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7373	AssertRCReturn(rc, rc);
7374
7375	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7376	rc = VBOXSTRICTRC_VAL(rc2);
7377	if (RT_LIKELY(rc == VINF_SUCCESS))
7378	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7379	else
7380	{
7381	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7382	pVmxTransient->uExitQualification, rc));
7383	}
7384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7385	return rc;
7386	}
7387
7388
7389	/**
7390	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7391	*/
7392	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7393	{
7394	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7395	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7396	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7397	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7398	AssertRCReturn(rc, rc);
7399
7400	PVM pVM = pVCpu->CTX_SUFF(pVM);
7401	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7402	if (RT_LIKELY(rc == VINF_SUCCESS))
7403	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7404	else
7405	{
7406	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7407	rc = VERR_EM_INTERPRETER;
7408	}
7409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7410	return rc;
7411	}
7412
7413
7414	/**
7415	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7416	*/
7417	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7418	{
7419	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7420	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7421	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7422	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7423	AssertRCReturn(rc, rc);
7424
7425	PVM pVM = pVCpu->CTX_SUFF(pVM);
7426	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7427	rc = VBOXSTRICTRC_VAL(rc2);
7428	if (RT_LIKELY( rc == VINF_SUCCESS
7429	\|\| rc == VINF_EM_HALT))
7430	{
7431	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7432	AssertRCReturn(rc3, rc3);
7433
7434	if ( rc == VINF_EM_HALT
7435	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7436	{
7437	rc = VINF_SUCCESS;
7438	}
7439	}
7440	else
7441	{
7442	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7443	rc = VERR_EM_INTERPRETER;
7444	}
7445	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7446	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7447	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7448	return rc;
7449	}
7450
7451
7452	/**
7453	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7454	*/
7455	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7456	{
7457	/*
7458	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7459	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7460	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7461	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7462	*/
7463	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7464	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7465	}
7466
7467
7468	/**
7469	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7470	*/
7471	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7472	{
7473	/*
7474	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7475	* root operation. If we get there there is something funny going on.
7476	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7477	*/
7478	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7479	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7480	}
7481
7482
7483	/**
7484	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7485	*/
7486	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7487	{
7488	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7489	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7490	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7491	}
7492
7493
7494	/**
7495	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7496	*/
7497	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7498	{
7499	/*
7500	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7501	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7502	* See Intel spec. 25.3 "Other Causes of VM-exits".
7503	*/
7504	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7505	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7506	}
7507
7508
7509	/**
7510	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7511	* VM-exit.
7512	*/
7513	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7514	{
7515	/*
7516	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7517	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7518	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7519	*/
7520	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7521	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7522	}
7523
7524
7525	/**
7526	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7527	* VM-exit.
7528	*/
7529	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7530	{
7531	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7532	return VINF_EM_RESET;
7533	}
7534
7535
7536	/**
7537	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7538	*/
7539	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7540	{
7541	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7542	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7543	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7544	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7545	AssertRCReturn(rc, rc);
7546
7547	pMixedCtx->rip++;
7548	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7549	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7550	rc = VINF_SUCCESS;
7551	else
7552	rc = VINF_EM_HALT;
7553
7554	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7555	return rc;
7556	}
7557
7558
7559	/**
7560	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7561	*/
7562	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7563	{
7564	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7565	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7566	return VINF_SUCCESS;
7567	}
7568
7569
7570	/**
7571	* VM-exit handler for expiry of the VMX preemption timer.
7572	*/
7573	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7574	{
7575	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7576
7577	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7578	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7579
7580	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7581	PVM pVM = pVCpu->CTX_SUFF(pVM);
7582	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7584	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7585	}
7586
7587
7588	/**
7589	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7590	*/
7591	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7592	{
7593	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7594
7595	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7596	/** @todo check if XSETBV is supported by the recompiler. */
7597	return VERR_EM_INTERPRETER;
7598	}
7599
7600
7601	/**
7602	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7603	*/
7604	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7605	{
7606	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7607
7608	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7609	/** @todo implement EMInterpretInvpcid() */
7610	return VERR_EM_INTERPRETER;
7611	}
7612
7613
7614	/**
7615	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7616	* Error VM-exit.
7617	*/
7618	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7619	{
7620	uint32_t uIntrState;
7621	HMVMXHCUINTREG uHCReg;
7622	uint64_t u64Val;
7623	uint32_t u32Val;
7624
7625	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7626	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7627	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7628	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7629	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7630	AssertRCReturn(rc, rc);
7631
7632	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7633	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7634	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7635	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7636
7637	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7638	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7639	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7640	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7641	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7642	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7643	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7644	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7645	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7646	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7647	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7648	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7649
7650	PVM pVM = pVCpu->CTX_SUFF(pVM);
7651	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7652
7653	return VERR_VMX_INVALID_GUEST_STATE;
7654	}
7655
7656
7657	/**
7658	* VM-exit handler for VM-entry failure due to an MSR-load
7659	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7660	*/
7661	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7662	{
7663	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7664	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7665	}
7666
7667
7668	/**
7669	* VM-exit handler for VM-entry failure due to a machine-check event
7670	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7671	*/
7672	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7673	{
7674	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7675	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7676	}
7677
7678
7679	/**
7680	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7681	* theory.
7682	*/
7683	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7684	{
7685	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7686	return VERR_VMX_UNDEFINED_EXIT_CODE;
7687	}
7688
7689
7690	/**
7691	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7692	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7693	* Conditional VM-exit.
7694	*/
7695	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7696	{
7697	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7698
7699	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7700	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7701	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7702	return VERR_EM_INTERPRETER;
7703	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7704	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7705	}
7706
7707
7708	/**
7709	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7710	*/
7711	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7712	{
7713	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7714
7715	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7717	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7718	return VERR_EM_INTERPRETER;
7719	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7720	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7721	}
7722
7723
7724	/**
7725	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7726	*/
7727	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7728	{
7729	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7730
7731	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7732	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7733	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7734	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7735	AssertRCReturn(rc, rc);
7736
7737	PVM pVM = pVCpu->CTX_SUFF(pVM);
7738	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7739	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7740	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7742
7743	if (RT_LIKELY(rc == VINF_SUCCESS))
7744	{
7745	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7746	Assert(pVmxTransient->cbInstr == 2);
7747	}
7748	return rc;
7749	}
7750
7751
7752	/**
7753	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7754	*/
7755	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7756	{
7757	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7758	PVM pVM = pVCpu->CTX_SUFF(pVM);
7759	int rc = VINF_SUCCESS;
7760
7761	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7762	if ( pVM->hm.s.fTPRPatchingActive
7763	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7764	{
7765	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7766	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7767	{
7768	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7769	AssertRC(rc);
7770	}
7771
7772	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7773	Assert(pVmxTransient->cbInstr == 2);
7774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7775	return VINF_SUCCESS;
7776	}
7777
7778	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7779	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7780	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7781	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7782	AssertRCReturn(rc, rc);
7783	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7784
7785	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7786	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7787	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7788
7789	if (RT_LIKELY(rc == VINF_SUCCESS))
7790	{
7791	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7792
7793	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7794	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7795	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7796	{
7797	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7798	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7799	}
7800	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7801	{
7802	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7803	AssertRCReturn(rc, rc);
7804	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7805	}
7806	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7807	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7808
7809	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7810	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7811	{
7812	switch (pMixedCtx->ecx)
7813	{
7814	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7815	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7816	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7817	case MSR_K8_FS_BASE: /* no break */
7818	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7819	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7820	}
7821	}
7822	#ifdef VBOX_STRICT
7823	else
7824	{
7825	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7826	switch (pMixedCtx->ecx)
7827	{
7828	case MSR_IA32_SYSENTER_CS:
7829	case MSR_IA32_SYSENTER_EIP:
7830	case MSR_IA32_SYSENTER_ESP:
7831	case MSR_K8_FS_BASE:
7832	case MSR_K8_GS_BASE:
7833	{
7834	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7835	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7836	}
7837
7838	case MSR_K8_LSTAR:
7839	case MSR_K6_STAR:
7840	case MSR_K8_SF_MASK:
7841	case MSR_K8_TSC_AUX:
7842	case MSR_K8_KERNEL_GS_BASE:
7843	{
7844	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7845	pMixedCtx->ecx));
7846	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7847	}
7848	}
7849	}
7850	#endif /* VBOX_STRICT */
7851	}
7852	return rc;
7853	}
7854
7855
7856	/**
7857	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7858	*/
7859	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7860	{
7861	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7862
7863	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7865	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7866	return VERR_EM_INTERPRETER;
7867	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7868	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7869	}
7870
7871
7872	/**
7873	* VM-exit handler for when the TPR value is lowered below the specified
7874	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7875	*/
7876	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7877	{
7878	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7879	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
7880
7881	/*
7882	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7883	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7884	* resume guest execution.
7885	*/
7886	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7888	return VINF_SUCCESS;
7889	}
7890
7891
7892	/**
7893	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7894	* VM-exit.
7895	*
7896	* @retval VINF_SUCCESS when guest execution can continue.
7897	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7898	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7899	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7900	* recompiler.
7901	*/
7902	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7903	{
7904	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7905	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7906	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7907	AssertRCReturn(rc, rc);
7908
7909	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7910	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7911	PVM pVM = pVCpu->CTX_SUFF(pVM);
7912	switch (uAccessType)
7913	{
7914	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7915	{
7916	#if 0
7917	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7918	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7919	#else
7920	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7921	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7922	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7923	#endif
7924	AssertRCReturn(rc, rc);
7925
7926	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7927	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7928	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7929	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7930
7931	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7932	{
7933	case 0: /* CR0 */
7934	Log(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
7935	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7936	break;
7937	case 2: /* C2 **/
7938	/* Nothing to do here, CR2 it's not part of the VMCS. */
7939	break;
7940	case 3: /* CR3 */
7941	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7942	Log(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
7943	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7944	break;
7945	case 4: /* CR4 */
7946	Log(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
7947	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7948	break;
7949	case 8: /* CR8 */
7950	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7951	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7952	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7953	break;
7954	default:
7955	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7956	break;
7957	}
7958
7959	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7960	break;
7961	}
7962
7963	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7964	{
7965	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7966	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7967	AssertRCReturn(rc, rc);
7968	Assert( !pVM->hm.s.fNestedPaging
7969	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7970	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7971
7972	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7973	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7974	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7975
7976	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7977	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7978	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7979	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7981	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7982	break;
7983	}
7984
7985	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7986	{
7987	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7988	AssertRCReturn(rc, rc);
7989	rc = EMInterpretCLTS(pVM, pVCpu);
7990	AssertRCReturn(rc, rc);
7991	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7993	Log(("CRX CLTS write rc=%d\n", rc));
7994	break;
7995	}
7996
7997	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7998	{
7999	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8000	AssertRCReturn(rc, rc);
8001	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8002	if (RT_LIKELY(rc == VINF_SUCCESS))
8003	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8004	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8005	Log(("CRX LMSW write rc=%d\n", rc));
8006	break;
8007	}
8008
8009	default:
8010	{
8011	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8012	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8013	}
8014	}
8015
8016	/* Validate possible error codes. */
8017	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8018	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8019	if (RT_SUCCESS(rc))
8020	{
8021	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8022	AssertRCReturn(rc2, rc2);
8023	}
8024
8025	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8026	return rc;
8027	}
8028
8029
8030	/**
8031	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8032	* VM-exit.
8033	*/
8034	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8035	{
8036	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8037	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8038
8039	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8040	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8041	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8042	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8043	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8044	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8045	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8046	AssertRCReturn(rc, rc);
8047	Log(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8048
8049	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8050	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8051	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8052	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8053	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8054	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8055	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8056
8057	/* I/O operation lookup arrays. */
8058	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8059	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8060
8061	const uint32_t cbSize = s_aIOSize[uIOWidth];
8062	const uint32_t cbInstr = pVmxTransient->cbInstr;
8063	PVM pVM = pVCpu->CTX_SUFF(pVM);
8064	if (fIOString)
8065	{
8066	/* INS/OUTS - I/O String instruction. */
8067	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8068	/** @todo for now manually disassemble later optimize by getting the fields from
8069	* the VMCS. */
8070	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8071	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8072	* segment prefix info. */
8073	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8074	if (RT_SUCCESS(rc))
8075	{
8076	if (fIOWrite)
8077	{
8078	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8079	(DISCPUMODE)pDis->uAddrMode, cbSize);
8080	rc = VBOXSTRICTRC_VAL(rc2);
8081	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8082	}
8083	else
8084	{
8085	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8086	(DISCPUMODE)pDis->uAddrMode, cbSize);
8087	rc = VBOXSTRICTRC_VAL(rc2);
8088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8089	}
8090	}
8091	else
8092	{
8093	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8094	rc = VINF_EM_RAW_EMULATE_INSTR;
8095	}
8096	}
8097	else
8098	{
8099	/* IN/OUT - I/O instruction. */
8100	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8101	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8102	if (fIOWrite)
8103	{
8104	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8105	rc = VBOXSTRICTRC_VAL(rc2);
8106	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8107	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8109	}
8110	else
8111	{
8112	uint32_t u32Result = 0;
8113	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8114	rc = VBOXSTRICTRC_VAL(rc2);
8115	if (IOM_SUCCESS(rc))
8116	{
8117	/* Save result of I/O IN instr. in AL/AX/EAX. */
8118	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8119	}
8120	else if (rc == VINF_IOM_R3_IOPORT_READ)
8121	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8123	}
8124	}
8125
8126	if (IOM_SUCCESS(rc))
8127	{
8128	pMixedCtx->rip += cbInstr;
8129	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8130	if (RT_LIKELY(rc == VINF_SUCCESS))
8131	{
8132	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8133	AssertRCReturn(rc, rc);
8134
8135	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8136	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8137	{
8138	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8139	for (unsigned i = 0; i < 4; i++)
8140	{
8141	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8142	if ( ( uIOPort >= pMixedCtx->dr[i]
8143	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8144	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8145	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8146	{
8147	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8148	uint64_t uDR6 = ASMGetDR6();
8149
8150	/* Clear all breakpoint status flags and set the one we just hit. */
8151	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8152	uDR6 \|= (uint64_t)RT_BIT(i);
8153
8154	/*
8155	* Note: AMD64 Architecture Programmer's Manual 13.1:
8156	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8157	* be cleared by software after the contents have been read.
8158	*/
8159	ASMSetDR6(uDR6);
8160
8161	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8162	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8163
8164	/* Paranoia. */
8165	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* Upper 32 bits MBZ. */
8166	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8167	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8168
8169	/* Resync DR7 */
8170	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8171	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8172
8173	/* Set #DB to be injected into the VM and continue guest execution. */
8174	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8175	break;
8176	}
8177	}
8178	}
8179	}
8180	}
8181
8182	#ifdef DEBUG
8183	if (rc == VINF_IOM_R3_IOPORT_READ)
8184	Assert(!fIOWrite);
8185	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8186	Assert(fIOWrite);
8187	else
8188	{
8189	AssertMsg( RT_FAILURE(rc)
8190	\|\| rc == VINF_SUCCESS
8191	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8192	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8193	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8194	}
8195	#endif
8196
8197	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8198	return rc;
8199	}
8200
8201
8202	/**
8203	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8204	* VM-exit.
8205	*/
8206	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8207	{
8208	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8209
8210	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8211	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8212	AssertRCReturn(rc, rc);
8213	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8214	{
8215	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8216	AssertRCReturn(rc, rc);
8217	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8218	{
8219	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8220	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8221	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8222	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8223	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8224	{
8225	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8226	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8227
8228	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8229	Assert(!pVCpu->hm.s.Event.fPending);
8230	pVCpu->hm.s.Event.fPending = true;
8231	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8232	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8233	AssertRCReturn(rc, rc);
8234	if (fErrorCodeValid)
8235	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8236	else
8237	pVCpu->hm.s.Event.u32ErrCode = 0;
8238	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8239	&& uVector == X86_XCPT_PF)
8240	{
8241	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8242	}
8243	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8244	}
8245	}
8246	}
8247	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8248	* emulation. */
8249	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8250	return VERR_EM_INTERPRETER;
8251	}
8252
8253
8254	/**
8255	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8256	*/
8257	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8258	{
8259	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8260	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8261	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8262	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8263	AssertRCReturn(rc, rc);
8264	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8265	return VINF_EM_DBG_STOP;
8266	}
8267
8268
8269	/**
8270	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8271	*/
8272	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8273	{
8274	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8275
8276	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8277	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8278	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8279	return VINF_SUCCESS;
8280	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8281	return rc;
8282
8283	#if 0
8284	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8285	* just sync the whole thing. */
8286	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8287	#else
8288	/* Aggressive state sync. for now. */
8289	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8290	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8291	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8292	#endif
8293	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8294	AssertRCReturn(rc, rc);
8295
8296	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8297	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8298	switch (uAccessType)
8299	{
8300	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8301	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8302	{
8303	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8304	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8305	{
8306	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8307	}
8308
8309	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8310	GCPhys &= PAGE_BASE_GC_MASK;
8311	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8312	PVM pVM = pVCpu->CTX_SUFF(pVM);
8313	Log(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8314	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8315
8316	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8317	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8318	CPUMCTX2CORE(pMixedCtx), GCPhys);
8319	rc = VBOXSTRICTRC_VAL(rc2);
8320	Log(("ApicAccess rc=%d\n", rc));
8321	if ( rc == VINF_SUCCESS
8322	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8323	\|\| rc == VERR_PAGE_NOT_PRESENT)
8324	{
8325	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8326	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8327	rc = VINF_SUCCESS;
8328	}
8329	break;
8330	}
8331
8332	default:
8333	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8334	rc = VINF_EM_RAW_EMULATE_INSTR;
8335	break;
8336	}
8337
8338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8339	return rc;
8340	}
8341
8342
8343	/**
8344	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8345	* VM-exit.
8346	*/
8347	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8348	{
8349	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8350
8351	/* We should -not- get this VM-exit if the guest is debugging. */
8352	if (CPUMIsGuestDebugStateActive(pVCpu))
8353	{
8354	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8355	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8356	}
8357
8358	int rc = VERR_INTERNAL_ERROR_5;
8359	if ( !DBGFIsStepping(pVCpu)
8360	&& !CPUMIsHyperDebugStateActive(pVCpu))
8361	{
8362	/* Don't intercept MOV DRx. */
8363	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8364	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8365	AssertRCReturn(rc, rc);
8366
8367	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8368	PVM pVM = pVCpu->CTX_SUFF(pVM);
8369	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8370	AssertRC(rc);
8371	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8372
8373	#ifdef VBOX_WITH_STATISTICS
8374	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8375	AssertRCReturn(rc, rc);
8376	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8377	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8378	else
8379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8380	#endif
8381	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8382	return VINF_SUCCESS;
8383	}
8384
8385	/*
8386	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8387	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8388	*/
8389	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8390	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8391	AssertRCReturn(rc, rc);
8392
8393	PVM pVM = pVCpu->CTX_SUFF(pVM);
8394	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8395	{
8396	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8397	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8398	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8399	if (RT_SUCCESS(rc))
8400	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8402	}
8403	else
8404	{
8405	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8406	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8407	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8409	}
8410
8411	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8412	if (RT_SUCCESS(rc))
8413	{
8414	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8415	AssertRCReturn(rc2, rc2);
8416	}
8417	return rc;
8418	}
8419
8420
8421	/**
8422	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8423	* Conditional VM-exit.
8424	*/
8425	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8426	{
8427	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8428	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8429
8430	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8431	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8432	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8433	return VINF_SUCCESS;
8434	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8435	return rc;
8436
8437	RTGCPHYS GCPhys = 0;
8438	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8439
8440	#if 0
8441	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8442	#else
8443	/* Aggressive state sync. for now. */
8444	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8445	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8446	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8447	#endif
8448	AssertRCReturn(rc, rc);
8449
8450	/*
8451	* If we succeed, resume guest execution.
8452	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8453	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8454	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8455	* weird case. See @bugref{6043}.
8456	*/
8457	PVM pVM = pVCpu->CTX_SUFF(pVM);
8458	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8459	rc = VBOXSTRICTRC_VAL(rc2);
8460	Log(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8461	if ( rc == VINF_SUCCESS
8462	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8463	\|\| rc == VERR_PAGE_NOT_PRESENT)
8464	{
8465	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8466	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8467	return VINF_SUCCESS;
8468	}
8469	return rc;
8470	}
8471
8472
8473	/**
8474	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8475	* VM-exit.
8476	*/
8477	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8478	{
8479	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8480	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8481
8482	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8483	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8484	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8485	return VINF_SUCCESS;
8486	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8487	return rc;
8488
8489	RTGCPHYS GCPhys = 0;
8490	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8491	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8492	#if 0
8493	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8494	#else
8495	/* Aggressive state sync. for now. */
8496	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8497	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8498	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8499	#endif
8500	AssertRCReturn(rc, rc);
8501
8502	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8503	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8504
8505	RTGCUINT uErrorCode = 0;
8506	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8507	uErrorCode \|= X86_TRAP_PF_ID;
8508	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8509	uErrorCode \|= X86_TRAP_PF_RW;
8510	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8511	uErrorCode \|= X86_TRAP_PF_P;
8512
8513	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8514
8515	Log(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8516	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8517
8518	/* Handle the pagefault trap for the nested shadow table. */
8519	PVM pVM = pVCpu->CTX_SUFF(pVM);
8520	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8521	TRPMResetTrap(pVCpu);
8522
8523	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8524	if ( rc == VINF_SUCCESS
8525	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8526	\|\| rc == VERR_PAGE_NOT_PRESENT)
8527	{
8528	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8529	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8530	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8531	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8532	return VINF_SUCCESS;
8533	}
8534
8535	Log(("EPT return to ring-3 rc=%d\n"));
8536	return rc;
8537	}
8538
8539
8540	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8541	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8542	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8543	/**
8544	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8545	*/
8546	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8547	{
8548	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8550
8551	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8552	AssertRCReturn(rc, rc);
8553
8554	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8555	{
8556	/* Old-style FPU error reporting needs some extra work. */
8557	/** @todo don't fall back to the recompiler, but do it manually. */
8558	return VERR_EM_INTERPRETER;
8559	}
8560	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8561	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8562	return rc;
8563	}
8564
8565
8566	/**
8567	* VM-exit exception handler for #BP (Breakpoint exception).
8568	*/
8569	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8570	{
8571	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8573
8574	/** @todo Try optimize this by not saving the entire guest state unless
8575	* really needed. */
8576	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8577	AssertRCReturn(rc, rc);
8578
8579	PVM pVM = pVCpu->CTX_SUFF(pVM);
8580	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8581	if (rc == VINF_EM_RAW_GUEST_TRAP)
8582	{
8583	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8584	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8585	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8586	AssertRCReturn(rc, rc);
8587
8588	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8589	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8590	}
8591
8592	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8593	return rc;
8594	}
8595
8596
8597	/**
8598	* VM-exit exception handler for #DB (Debug exception).
8599	*/
8600	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8601	{
8602	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8604
8605	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8606	AssertRCReturn(rc, rc);
8607
8608	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8609	uint64_t uDR6 = X86_DR6_INIT_VAL;
8610	uDR6 \|= (pVmxTransient->uExitQualification
8611	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8612	PVM pVM = pVCpu->CTX_SUFF(pVM);
8613	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8614	if (rc == VINF_EM_RAW_GUEST_TRAP)
8615	{
8616	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8617	pMixedCtx->dr[6] = uDR6;
8618
8619	if (CPUMIsGuestDebugStateActive(pVCpu))
8620	ASMSetDR6(pMixedCtx->dr[6]);
8621
8622	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8623
8624	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8625	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8626
8627	/* Paranoia. */
8628	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* upper 32 bits MBZ. */
8629	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8630	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8631
8632	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8633	AssertRCReturn(rc,rc);
8634
8635	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8636	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8637	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8638	AssertRCReturn(rc2, rc2);
8639	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8640	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8641	rc = VINF_SUCCESS;
8642	}
8643
8644	return rc;
8645	}
8646
8647
8648	/**
8649	* VM-exit exception handler for #NM (Device-not-available exception: floating
8650	* point exception).
8651	*/
8652	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8653	{
8654	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8655
8656	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8657	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8658	#endif
8659
8660	/* We require CR0 and EFER. EFER is always up-to-date. */
8661	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8662	AssertRCReturn(rc, rc);
8663
8664	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8665	PVM pVM = pVCpu->CTX_SUFF(pVM);
8666	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8667	if (rc == VINF_SUCCESS)
8668	{
8669	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8670	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8671	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8672	return VINF_SUCCESS;
8673	}
8674
8675	/* Forward #NM to the guest. */
8676	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8677	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8678	AssertRCReturn(rc, rc);
8679	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8680	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8681	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8682	return rc;
8683	}
8684
8685
8686	/**
8687	* VM-exit exception handler for #GP (General-protection exception).
8688	*
8689	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8690	*/
8691	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8692	{
8693	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8694	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8695
8696	int rc = VERR_INTERNAL_ERROR_5;
8697	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8698	{
8699	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8700	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8701	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8702	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8703	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8704	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8705	AssertRCReturn(rc, rc);
8706	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8707	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8708	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8709	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8710	return rc;
8711	#else
8712	/* We don't intercept #GP. */
8713	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8714	return VERR_VMX_UNEXPECTED_EXCEPTION;
8715	#endif
8716	}
8717
8718	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8719	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8720
8721	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8722	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8723	AssertRCReturn(rc, rc);
8724
8725	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8726	uint32_t cbOp = 0;
8727	PVM pVM = pVCpu->CTX_SUFF(pVM);
8728	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8729	if (RT_SUCCESS(rc))
8730	{
8731	rc = VINF_SUCCESS;
8732	Assert(cbOp == pDis->cbInstr);
8733	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8734	switch (pDis->pCurInstr->uOpcode)
8735	{
8736	case OP_CLI:
8737	pMixedCtx->eflags.Bits.u1IF = 0;
8738	pMixedCtx->rip += pDis->cbInstr;
8739	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8741	break;
8742
8743	case OP_STI:
8744	pMixedCtx->eflags.Bits.u1IF = 1;
8745	pMixedCtx->rip += pDis->cbInstr;
8746	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8747	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8748	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8749	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8750	break;
8751
8752	case OP_HLT:
8753	rc = VINF_EM_HALT;
8754	pMixedCtx->rip += pDis->cbInstr;
8755	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8757	break;
8758
8759	case OP_POPF:
8760	{
8761	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8762	uint32_t cbParm = 0;
8763	uint32_t uMask = 0;
8764	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8765	{
8766	cbParm = 4;
8767	uMask = 0xffffffff;
8768	}
8769	else
8770	{
8771	cbParm = 2;
8772	uMask = 0xffff;
8773	}
8774
8775	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8776	RTGCPTR GCPtrStack = 0;
8777	X86EFLAGS uEflags;
8778	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8779	&GCPtrStack);
8780	if (RT_SUCCESS(rc))
8781	{
8782	Assert(sizeof(uEflags.u32) >= cbParm);
8783	uEflags.u32 = 0;
8784	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8785	}
8786	if (RT_FAILURE(rc))
8787	{
8788	rc = VERR_EM_INTERPRETER;
8789	break;
8790	}
8791	Log(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8792	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8793	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8794	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8795	pMixedCtx->eflags.Bits.u1RF = 0;
8796	pMixedCtx->esp += cbParm;
8797	pMixedCtx->esp &= uMask;
8798	pMixedCtx->rip += pDis->cbInstr;
8799	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8801	break;
8802	}
8803
8804	case OP_PUSHF:
8805	{
8806	uint32_t cbParm = 0;
8807	uint32_t uMask = 0;
8808	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8809	{
8810	cbParm = 4;
8811	uMask = 0xffffffff;
8812	}
8813	else
8814	{
8815	cbParm = 2;
8816	uMask = 0xffff;
8817	}
8818
8819	/* Get the stack pointer & push the contents of eflags onto the stack. */
8820	RTGCPTR GCPtrStack = 0;
8821	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8822	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8823	if (RT_FAILURE(rc))
8824	{
8825	rc = VERR_EM_INTERPRETER;
8826	break;
8827	}
8828	X86EFLAGS uEflags;
8829	uEflags = pMixedCtx->eflags;
8830	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8831	uEflags.Bits.u1RF = 0;
8832	uEflags.Bits.u1VM = 0;
8833
8834	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8835	if (RT_FAILURE(rc))
8836	{
8837	rc = VERR_EM_INTERPRETER;
8838	break;
8839	}
8840	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8841	pMixedCtx->esp -= cbParm;
8842	pMixedCtx->esp &= uMask;
8843	pMixedCtx->rip += pDis->cbInstr;
8844	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8846	break;
8847	}
8848
8849	case OP_IRET:
8850	{
8851	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8852	* instruction reference. */
8853	RTGCPTR GCPtrStack = 0;
8854	uint32_t uMask = 0xffff;
8855	uint16_t aIretFrame[3];
8856	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8857	{
8858	rc = VERR_EM_INTERPRETER;
8859	break;
8860	}
8861	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8862	&GCPtrStack);
8863	if (RT_SUCCESS(rc))
8864	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8865	if (RT_FAILURE(rc))
8866	{
8867	rc = VERR_EM_INTERPRETER;
8868	break;
8869	}
8870	pMixedCtx->eip = 0;
8871	pMixedCtx->ip = aIretFrame[0];
8872	pMixedCtx->cs.Sel = aIretFrame[1];
8873	pMixedCtx->cs.ValidSel = aIretFrame[1];
8874	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8875	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8876	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8877	pMixedCtx->sp += sizeof(aIretFrame);
8878	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8879	\| HM_CHANGED_GUEST_RFLAGS;
8880	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8881	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8882	break;
8883	}
8884
8885	case OP_INT:
8886	{
8887	uint16_t uVector = pDis->Param1.uValue & 0xff;
8888	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8890	break;
8891	}
8892
8893	case OP_INTO:
8894	{
8895	if (pMixedCtx->eflags.Bits.u1OF)
8896	{
8897	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8899	}
8900	break;
8901	}
8902
8903	default:
8904	{
8905	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8906	EMCODETYPE_SUPERVISOR);
8907	rc = VBOXSTRICTRC_VAL(rc2);
8908	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8909	Log(("#GP rc=%Rrc\n", rc));
8910	break;
8911	}
8912	}
8913	}
8914	else
8915	rc = VERR_EM_INTERPRETER;
8916
8917	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8918	("#GP Unexpected rc=%Rrc\n", rc));
8919	return rc;
8920	}
8921
8922
8923	/**
8924	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8925	* the exception reported in the VMX transient structure back into the VM.
8926	*
8927	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8928	* up-to-date.
8929	*/
8930	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8931	{
8932	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8933
8934	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8935	hmR0VmxCheckExitDueToEventDelivery(). */
8936	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8937	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8938	AssertRCReturn(rc, rc);
8939	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8940	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8941	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8942	return VINF_SUCCESS;
8943	}
8944
8945
8946	/**
8947	* VM-exit exception handler for #PF (Page-fault exception).
8948	*/
8949	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8950	{
8951	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8952	PVM pVM = pVCpu->CTX_SUFF(pVM);
8953	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8954	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8955	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8956	AssertRCReturn(rc, rc);
8957
8958	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8959	if (pVM->hm.s.fNestedPaging)
8960	{
8961	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8962	{
8963	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8964	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8965	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8966	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8967	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8968	}
8969	else
8970	{
8971	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8972	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8973	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8974	Log(("Pending #DF due to vectoring #PF. NP\n"));
8975	}
8976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8977	return rc;
8978	}
8979	#else
8980	Assert(!pVM->hm.s.fNestedPaging);
8981	#endif
8982
8983	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8984	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8985	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8986	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8987	AssertRCReturn(rc, rc);
8988	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8989	if ( pVM->hm.s.fTRPPatchingAllowed
8990	&& pVM->hm.s.pGuestPatchMem
8991	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8992	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8993	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8994	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8995	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8996	{
8997	RTGCPHYS GCPhys;
8998	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8999	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9000	if ( rc == VINF_SUCCESS
9001	&& GCPhys == GCPhysApicBase)
9002	{
9003	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9004	AssertRCReturn(rc, rc);
9005
9006	/* Only attempt to patch the instruction once. */
9007	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9008	if (!pPatch)
9009	return VINF_EM_HM_PATCH_TPR_INSTR;
9010	}
9011	}
9012	#endif
9013
9014	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9015	AssertRCReturn(rc, rc);
9016
9017	Log(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
9018	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9019
9020	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9021	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9022	(RTGCPTR)pVmxTransient->uExitQualification);
9023
9024	Log(("#PF: rc=%Rrc\n", rc));
9025	if (rc == VINF_SUCCESS)
9026	{
9027	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9028	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9029	* memory? We don't update the whole state here... */
9030	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9031	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9032	TRPMResetTrap(pVCpu);
9033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9034	return rc;
9035	}
9036	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9037	{
9038	if (!pVmxTransient->fVectoringPF)
9039	{
9040	/* It's a guest page fault and needs to be reflected to the guest. */
9041	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9042	TRPMResetTrap(pVCpu);
9043	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9044	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9045	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9046	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9047	}
9048	else
9049	{
9050	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9051	TRPMResetTrap(pVCpu);
9052	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9053	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9054	Log(("#PF: Pending #DF due to vectoring #PF\n"));
9055	}
9056
9057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9058	return VINF_SUCCESS;
9059	}
9060
9061	TRPMResetTrap(pVCpu);
9062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9063	return rc;
9064	}
9065

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46130

Download in other formats: