HMVMXR0.cpp@ 45908

Last change on this file since 45908 was 45908, checked in by vboxsync, 12 years ago
Don't OR status codes together, just AssertRCReturn immediately. If two methods fails with different statuses, we'll be very confused. Also, for (human) analysis, return VINF_SUCCESS when you mean no informational statuses are used. int rc = VINF_SUCCESS; at the top and return rc; at the bottom might save an xor eax,eax; instruction in the slow path, but it makes it harder to read. (It also forces the compiler to remember the last rc value, which might not come without a cost on register starved 32-bit.)
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 376.6 KB

Line
1	/* $Id: HMVMXR0.cpp 45908 2013-05-06 12:59:37Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147	/** Profiling macro. */
148	#ifdef HM_PROFILE_EXIT_DISPATCH
149	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
150	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
151	#else
152	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
153	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
154	#endif
155
156
157	/*******************************************************************************
158	* Structures and Typedefs *
159	*******************************************************************************/
160	/**
161	* A state structure for holding miscellaneous information across
162	* VMX non-root operation and restored after the transition.
163	*/
164	typedef struct VMXTRANSIENT
165	{
166	/** The host's rflags/eflags. */
167	RTCCUINTREG uEFlags;
168	#if HC_ARCH_BITS == 32
169	uint32_t u32Alignment0;
170	#endif
171	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
172	uint64_t u64LStarMsr;
173	/** The guest's TPR value used for TPR shadowing. */
174	uint8_t u8GuestTpr;
175	/** Alignment. */
176	uint8_t abAlignment0[6];
177
178	/** The basic VM-exit reason. */
179	uint16_t uExitReason;
180	/** Alignment. */
181	uint16_t u16Alignment0;
182	/** The VM-exit interruption error code. */
183	uint32_t uExitIntrErrorCode;
184	/** The VM-exit exit qualification. */
185	RTGCUINTPTR uExitQualification;
186	#if GC_ARCH_BITS == 32
187	/** Alignment. */
188	uint32_t u32Alignment1;
189	#endif
190
191	/** The VM-exit interruption-information field. */
192	uint32_t uExitIntrInfo;
193	/** The VM-exit instruction-length field. */
194	uint32_t cbInstr;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Alignment. */
198	uint8_t abAlignment1[5];
199
200	/** The VM-entry interruption-information field. */
201	uint32_t uEntryIntrInfo;
202	/** The VM-entry exception error code field. */
203	uint32_t uEntryXcptErrorCode;
204	/** The VM-entry instruction length field. */
205	uint32_t cbEntryInstr;
206
207	/** IDT-vectoring information field. */
208	uint32_t uIdtVectoringInfo;
209	/** IDT-vectoring error code. */
210	uint32_t uIdtVectoringErrorCode;
211
212	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
213	uint32_t fVmcsFieldsRead;
214	/** Whether TSC-offsetting should be setup before VM-entry. */
215	bool fUpdateTscOffsettingAndPreemptTimer;
216	/** Whether the VM-exit was caused by a page-fault during delivery of a
217	* contributary exception or a page-fault. */
218	bool fVectoringPF;
219	} VMXTRANSIENT, *PVMXTRANSIENT;
220	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
221	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
222	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
223
224
225	/**
226	* MSR-bitmap read permissions.
227	*/
228	typedef enum VMXMSREXITREAD
229	{
230	/** Reading this MSR causes a VM-exit. */
231	VMXMSREXIT_INTERCEPT_READ = 0xb,
232	/** Reading this MSR does not cause a VM-exit. */
233	VMXMSREXIT_PASSTHRU_READ
234	} VMXMSREXITREAD;
235
236	/**
237	* MSR-bitmap write permissions.
238	*/
239	typedef enum VMXMSREXITWRITE
240	{
241	/** Writing to this MSR causes a VM-exit. */
242	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
243	/** Writing to this MSR does not cause a VM-exit. */
244	VMXMSREXIT_PASSTHRU_WRITE
245	} VMXMSREXITWRITE;
246
247
248	/*******************************************************************************
249	* Internal Functions *
250	*******************************************************************************/
251	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
252	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
253	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
254	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
255	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
256	#endif
257	#ifndef HMVMX_USE_FUNCTION_TABLE
258	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
259	#define HMVMX_EXIT_DECL static int
260	#else
261	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
262	#endif
263
264	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308
309	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316
317
318	/*******************************************************************************
319	* Global Variables *
320	*******************************************************************************/
321	#ifdef HMVMX_USE_FUNCTION_TABLE
322	/**
323	* VM-exit handler.
324	*
325	* @returns VBox status code.
326	* @param pVCpu Pointer to the VMCPU.
327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
328	* out-of-sync. Make sure to update the required
329	* fields before using them.
330	* @param pVmxTransient Pointer to the VMX-transient structure.
331	*/
332	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333	/** Pointer to VM-exit handler. */
334	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
335
336	/**
337	* VMX_EXIT dispatch table.
338	*/
339	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
340	{
341	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
342	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
343	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
344	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
345	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
346	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
347	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
348	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
349	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
350	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
351	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
352	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
353	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
354	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
355	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
356	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
357	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
358	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
359	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
360	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
361	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
362	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
363	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
364	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
365	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
366	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
367	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
368	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
369	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
370	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
371	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
372	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
373	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
374	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
375	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
376	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
377	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
378	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
379	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
380	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
381	/* 40 UNDEFINED */ hmR0VmxExitPause,
382	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
383	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
384	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
385	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
386	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
387	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
388	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
389	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
390	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
391	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
392	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
393	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
394	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
395	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
396	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
397	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
398	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
399	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
400	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
401	};
402	#endif /* HMVMX_USE_FUNCTION_TABLE */
403
404	#ifdef VBOX_STRICT
405	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
406	{
407	/* 0 */ "(Not Used)",
408	/* 1 */ "VMCALL executed in VMX root operation.",
409	/* 2 */ "VMCLEAR with invalid physical address.",
410	/* 3 */ "VMCLEAR with VMXON pointer.",
411	/* 4 */ "VMLAUNCH with non-clear VMCS.",
412	/* 5 */ "VMRESUME with non-launched VMCS.",
413	/* 6 */ "VMRESUME after VMXOFF",
414	/* 7 */ "VM entry with invalid control fields.",
415	/* 8 */ "VM entry with invalid host state fields.",
416	/* 9 */ "VMPTRLD with invalid physical address.",
417	/* 10 */ "VMPTRLD with VMXON pointer.",
418	/* 11 */ "VMPTRLD with incorrect revision identifier.",
419	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
420	/* 13 */ "VMWRITE to read-only VMCS component.",
421	/* 14 */ "(Not Used)",
422	/* 15 */ "VMXON executed in VMX root operation.",
423	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
424	/* 17 */ "VM entry with non-launched executing VMCS.",
425	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
426	/* 19 */ "VMCALL with non-clear VMCS.",
427	/* 20 */ "VMCALL with invalid VM-exit control fields.",
428	/* 21 */ "(Not Used)",
429	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
430	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
431	/* 24 */ "VMCALL with invalid SMM-monitor features.",
432	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
433	/* 26 */ "VM entry with events blocked by MOV SS.",
434	/* 27 */ "(Not Used)",
435	/* 28 */ "Invalid operand to INVEPT/INVVPID."
436	};
437	#endif /* VBOX_STRICT */
438
439
440
441	/**
442	* Updates the VM's last error record. If there was a VMX instruction error,
443	* reads the error data from the VMCS and updates VCPU's last error record as
444	* well.
445	*
446	* @param pVM Pointer to the VM.
447	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
448	* VERR_VMX_UNABLE_TO_START_VM or
449	* VERR_VMX_INVALID_VMCS_FIELD).
450	* @param rc The error code.
451	*/
452	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
453	{
454	AssertPtr(pVM);
455	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
456	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
457	{
458	AssertPtrReturnVoid(pVCpu);
459	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
460	}
461	pVM->hm.s.lLastError = rc;
462	}
463
464
465	/**
466	* Reads the VM-entry interruption-information field from the VMCS into the VMX
467	* transient structure.
468	*
469	* @returns VBox status code.
470	* @param pVmxTransient Pointer to the VMX transient structure.
471	*
472	* @remarks No-long-jump zone!!!
473	*/
474	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
475	{
476	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
477	AssertRCReturn(rc, rc);
478	return VINF_SUCCESS;
479	}
480
481
482	/**
483	* Reads the VM-entry exception error code field from the VMCS into
484	* the VMX transient structure.
485	*
486	* @returns VBox status code.
487	* @param pVmxTransient Pointer to the VMX transient structure.
488	*
489	* @remarks No-long-jump zone!!!
490	*/
491	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
492	{
493	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
494	AssertRCReturn(rc, rc);
495	return VINF_SUCCESS;
496	}
497
498
499	/**
500	* Reads the VM-entry exception error code field from the VMCS into
501	* the VMX transient structure.
502	*
503	* @returns VBox status code.
504	* @param pVCpu Pointer to the VMCPU.
505	* @param pVmxTransient Pointer to the VMX transient structure.
506	*
507	* @remarks No-long-jump zone!!!
508	*/
509	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
510	{
511	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
512	AssertRCReturn(rc, rc);
513	return VINF_SUCCESS;
514	}
515
516
517	/**
518	* Reads the VM-exit interruption-information field from the VMCS into the VMX
519	* transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
528	{
529	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the VM-exit interruption error code from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVCpu Pointer to the VMCPU.
543	* @param pVmxTransient Pointer to the VMX transient structure.
544	*/
545	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
546	{
547	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
548	{
549	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
550	AssertRCReturn(rc, rc);
551	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
552	}
553	return VINF_SUCCESS;
554	}
555
556
557	/**
558	* Reads the VM-exit instruction length field from the VMCS into the VMX
559	* transient structure.
560	*
561	* @returns VBox status code.
562	* @param pVCpu Pointer to the VMCPU.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Reads the exit qualification from the VMCS into the VMX transient structure.
579	*
580	* @returns VBox status code.
581	* @param pVCpu Pointer to the VMCPU.
582	* @param pVmxTransient Pointer to the VMX transient structure.
583	*/
584	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
585	{
586	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
587	{
588	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
589	AssertRCReturn(rc, rc);
590	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
591	}
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the IDT-vectoring information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*
603	* @remarks No-long-jump zone!!!
604	*/
605	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
606	{
607	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
608	{
609	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
610	AssertRCReturn(rc, rc);
611	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
612	}
613	return VINF_SUCCESS;
614	}
615
616
617	/**
618	* Reads the IDT-vectoring error code from the VMCS into the VMX
619	* transient structure.
620	*
621	* @returns VBox status code.
622	* @param pVmxTransient Pointer to the VMX transient structure.
623	*/
624	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
625	{
626	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
627	{
628	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
629	AssertRCReturn(rc, rc);
630	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
631	}
632	return VINF_SUCCESS;
633	}
634
635
636	/**
637	* Enters VMX root mode operation on the current CPU.
638	*
639	* @returns VBox status code.
640	* @param pVM Pointer to the VM (optional, can be NULL, after
641	* a resume).
642	* @param HCPhysCpuPage Physical address of the VMXON region.
643	* @param pvCpuPage Pointer to the VMXON region.
644	*/
645	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
646	{
647	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
648	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
649	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
650
651	if (pVM)
652	{
653	/* Write the VMCS revision dword to the VMXON region. */
654	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
655	}
656
657	/* Enable the VMX bit in CR4 if necessary. */
658	RTCCUINTREG uCr4 = ASMGetCR4();
659	if (!(uCr4 & X86_CR4_VMXE))
660	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
661
662	/* Enter VMX root mode. */
663	int rc = VMXEnable(HCPhysCpuPage);
664	if (RT_FAILURE(rc))
665	ASMSetCR4(uCr4);
666
667	return rc;
668	}
669
670
671	/**
672	* Exits VMX root mode operation on the current CPU.
673	*
674	* @returns VBox status code.
675	*/
676	static int hmR0VmxLeaveRootMode(void)
677	{
678	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
679
680	/* If we're for some reason not in VMX root mode, then don't leave it. */
681	if (ASMGetCR4() & X86_CR4_VMXE)
682	{
683	/* Exit VMX root mode and clear the VMX bit in CR4 */
684	VMXDisable();
685	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
686	return VINF_SUCCESS;
687	}
688
689	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
690	}
691
692
693	/**
694	* Allocates and maps one physically contiguous page. The allocated page is
695	* zero'd out. (Used by various VT-x structures).
696	*
697	* @returns IPRT status code.
698	* @param pMemObj Pointer to the ring-0 memory object.
699	* @param ppVirt Where to store the virtual address of the
700	* allocation.
701	* @param pPhys Where to store the physical address of the
702	* allocation.
703	*/
704	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
705	{
706	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
707	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
708	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
709
710	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
711	if (RT_FAILURE(rc))
712	return rc;
713	ppVirt = RTR0MemObjAddress(pMemObj);
714	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
715	ASMMemZero32(*ppVirt, PAGE_SIZE);
716	return VINF_SUCCESS;
717	}
718
719
720	/**
721	* Frees and unmaps an allocated physical page.
722	*
723	* @param pMemObj Pointer to the ring-0 memory object.
724	* @param ppVirt Where to re-initialize the virtual address of
725	* allocation as 0.
726	* @param pHCPhys Where to re-initialize the physical address of the
727	* allocation as 0.
728	*/
729	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
730	{
731	AssertPtr(pMemObj);
732	AssertPtr(ppVirt);
733	AssertPtr(pHCPhys);
734	if (*pMemObj != NIL_RTR0MEMOBJ)
735	{
736	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
737	AssertRC(rc);
738	*pMemObj = NIL_RTR0MEMOBJ;
739	*ppVirt = 0;
740	*pHCPhys = 0;
741	}
742	}
743
744
745	/**
746	* Worker function to free VT-x related structures.
747	*
748	* @returns IPRT status code.
749	* @param pVM Pointer to the VM.
750	*/
751	static void hmR0VmxStructsFree(PVM pVM)
752	{
753	for (VMCPUID i = 0; i < pVM->cCpus; i++)
754	{
755	PVMCPU pVCpu = &pVM->aCpus[i];
756	AssertPtr(pVCpu);
757
758	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
759	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
760	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
761	#endif
762
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
765
766	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
768	}
769
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
772	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
773	#endif
774	}
775
776
777	/**
778	* Worker function to allocate VT-x related VM structures.
779	*
780	* @returns IPRT status code.
781	* @param pVM Pointer to the VM.
782	*/
783	static int hmR0VmxStructsAlloc(PVM pVM)
784	{
785	/*
786	* Initialize members up-front so we can cleanup properly on allocation failure.
787	*/
788	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
789	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
790	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
791	pVM->hm.s.vmx.HCPhys##a_Name = 0;
792
793	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
794	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
795	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
796	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
797
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
800	#endif
801	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
802
803	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
804	for (VMCPUID i = 0; i < pVM->cCpus; i++)
805	{
806	PVMCPU pVCpu = &pVM->aCpus[i];
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
808	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
810	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
813	#endif
814	}
815	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
816	#undef VMXLOCAL_INIT_VM_MEMOBJ
817
818	/*
819	* Allocate all the VT-x structures.
820	*/
821	int rc = VINF_SUCCESS;
822	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
823	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
827	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
828	#endif
829
830	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
831	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
832	{
833	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
834	&pVM->hm.s.vmx.HCPhysApicAccess);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	/*
840	* Initialize per-VCPU VT-x structures.
841	*/
842	for (VMCPUID i =0; i < pVM->cCpus; i++)
843	{
844	PVMCPU pVCpu = &pVM->aCpus[i];
845	AssertPtr(pVCpu);
846
847	/* Allocate the VM control structure (VMCS). */
848	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
854	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
855	{
856	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
857	&pVCpu->hm.s.vmx.HCPhysVirtApic);
858	if (RT_FAILURE(rc))
859	goto cleanup;
860	}
861
862	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
863	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
864	{
865	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
866	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
867	if (RT_FAILURE(rc))
868	goto cleanup;
869	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
870	}
871
872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
873	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
874	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875	if (RT_FAILURE(rc))
876	goto cleanup;
877
878	/* Allocate the VM-exit MSR-load page for the host MSRs. */
879	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
880	if (RT_FAILURE(rc))
881	goto cleanup;
882	#endif
883	}
884
885	return VINF_SUCCESS;
886
887	cleanup:
888	hmR0VmxStructsFree(pVM);
889	return rc;
890	}
891
892
893	/**
894	* Does global VT-x initialization (called during module initialization).
895	*
896	* @returns VBox status code.
897	*/
898	VMMR0DECL(int) VMXR0GlobalInit(void)
899	{
900	#ifdef HMVMX_USE_FUNCTION_TABLE
901	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
902	# ifdef VBOX_STRICT
903	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
904	Assert(g_apfnVMExitHandlers[i]);
905	# endif
906	#endif
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Does global VT-x termination (called during module termination).
913	*/
914	VMMR0DECL(void) VMXR0GlobalTerm()
915	{
916	/* Nothing to do currently. */
917	}
918
919
920	/**
921	* Sets up and activates VT-x on the current CPU.
922	*
923	* @returns VBox status code.
924	* @param pCpu Pointer to the global CPU info struct.
925	* @param pVM Pointer to the VM (can be NULL after a host resume
926	* operation).
927	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
928	* fEnabledByHost is true).
929	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
930	* @a fEnabledByHost is true).
931	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
932	* enable VT-x/AMD-V on the host.
933	*/
934	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
935	{
936	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
937	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
938
939	if (!fEnabledByHost)
940	{
941	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
942	if (RT_FAILURE(rc))
943	return rc;
944	}
945
946	/*
947	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
948	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
949	* each time while reusing a VPID after hitting the MaxASID limit once.
950	*/
951	if ( pVM
952	&& pVM->hm.s.vmx.fVpid
953	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
954	{
955	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
956	pCpu->fFlushAsidBeforeUse = false;
957	}
958	else
959	pCpu->fFlushAsidBeforeUse = true;
960
961	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
962	++pCpu->cTlbFlushes;
963
964	return VINF_SUCCESS;
965	}
966
967
968	/**
969	* Deactivates VT-x on the current CPU.
970	*
971	* @returns VBox status code.
972	* @param pCpu Pointer to the global CPU info struct.
973	* @param pvCpuPage Pointer to the VMXON region.
974	* @param HCPhysCpuPage Physical address of the VMXON region.
975	*/
976	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
977	{
978	NOREF(pCpu);
979	NOREF(pvCpuPage);
980	NOREF(HCPhysCpuPage);
981
982	return hmR0VmxLeaveRootMode();
983	}
984
985
986	/**
987	* Sets the permission bits for the specified MSR in the MSR bitmap.
988	*
989	* @param pVCpu Pointer to the VMCPU.
990	* @param uMSR The MSR value.
991	* @param enmRead Whether reading this MSR causes a VM-exit.
992	* @param enmWrite Whether writing this MSR causes a VM-exit.
993	*/
994	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
995	{
996	int32_t iBit;
997	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
998
999	/*
1000	* Layout:
1001	* 0x000 - 0x3ff - Low MSR read bits
1002	* 0x400 - 0x7ff - High MSR read bits
1003	* 0x800 - 0xbff - Low MSR write bits
1004	* 0xc00 - 0xfff - High MSR write bits
1005	*/
1006	if (uMsr <= 0x00001FFF)
1007	iBit = uMsr;
1008	else if ( uMsr >= 0xC0000000
1009	&& uMsr <= 0xC0001FFF)
1010	{
1011	iBit = (uMsr - 0xC0000000);
1012	pbMsrBitmap += 0x400;
1013	}
1014	else
1015	{
1016	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1017	return;
1018	}
1019
1020	Assert(iBit <= 0x1fff);
1021	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1022	ASMBitSet(pbMsrBitmap, iBit);
1023	else
1024	ASMBitClear(pbMsrBitmap, iBit);
1025
1026	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1027	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1028	else
1029	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1030	}
1031
1032
1033	/**
1034	* Flushes the TLB using EPT.
1035	*
1036	* @returns VBox status code.
1037	* @param pVM Pointer to the VM.
1038	* @param pVCpu Pointer to the VMCPU.
1039	* @param enmFlush Type of flush.
1040	*/
1041	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1042	{
1043	AssertPtr(pVM);
1044	Assert(pVM->hm.s.fNestedPaging);
1045
1046	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1047
1048	uint64_t descriptor[2];
1049	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1050	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1051
1052	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1053	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1054	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1055	}
1056
1057
1058	/**
1059	* Flushes the TLB using VPID.
1060	*
1061	* @returns VBox status code.
1062	* @param pVM Pointer to the VM.
1063	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1064	* enmFlush).
1065	* @param enmFlush Type of flush.
1066	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1067	* on @a enmFlush).
1068	*/
1069	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1070	{
1071	AssertPtr(pVM);
1072	Assert(pVM->hm.s.vmx.fVpid);
1073
1074	uint64_t descriptor[2];
1075	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1076	{
1077	descriptor[0] = 0;
1078	descriptor[1] = 0;
1079	}
1080	else
1081	{
1082	AssertPtr(pVCpu);
1083	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1084	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1085	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1086	descriptor[1] = GCPtr;
1087	}
1088
1089	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1090	AssertMsg(rc == VINF_SUCCESS,
1091	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1092	if ( RT_SUCCESS(rc)
1093	&& pVCpu)
1094	{
1095	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1096	}
1097	}
1098
1099
1100	/**
1101	* Invalidates a guest page by guest virtual address. Only relevant for
1102	* EPT/VPID, otherwise there is nothing really to invalidate.
1103	*
1104	* @returns VBox status code.
1105	* @param pVM Pointer to the VM.
1106	* @param pVCpu Pointer to the VMCPU.
1107	* @param GCVirt Guest virtual address of the page to invalidate.
1108	*/
1109	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1110	{
1111	AssertPtr(pVM);
1112	AssertPtr(pVCpu);
1113	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1114
1115	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1116	if (!fFlushPending)
1117	{
1118	/*
1119	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1120	* See @bugref{6043} and @bugref{6177}.
1121	*
1122	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1123	* function maybe called in a loop with individual addresses.
1124	*/
1125	if (pVM->hm.s.vmx.fVpid)
1126	{
1127	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1128	{
1129	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1130	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1131	}
1132	else
1133	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1134	}
1135	else if (pVM->hm.s.fNestedPaging)
1136	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1137	}
1138
1139	return VINF_SUCCESS;
1140	}
1141
1142
1143	/**
1144	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1145	* otherwise there is nothing really to invalidate.
1146	*
1147	* @returns VBox status code.
1148	* @param pVM Pointer to the VM.
1149	* @param pVCpu Pointer to the VMCPU.
1150	* @param GCPhys Guest physical address of the page to invalidate.
1151	*/
1152	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1153	{
1154	LogFlowFunc(("%RGp\n", GCPhys));
1155
1156	/*
1157	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1158	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1159	* This function might be called in a loop.
1160	*/
1161	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1163	return VINF_SUCCESS;
1164	}
1165
1166
1167	/**
1168	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1169	* case where neither EPT nor VPID is supported by the CPU.
1170	*
1171	* @param pVM Pointer to the VM.
1172	* @param pVCpu Pointer to the VMCPU.
1173	*
1174	* @remarks Called with interrupts disabled.
1175	*/
1176	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1177	{
1178	NOREF(pVM);
1179	AssertPtr(pVCpu);
1180	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1181	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1182
1183	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1184	AssertPtr(pCpu);
1185
1186	pVCpu->hm.s.TlbShootdown.cPages = 0;
1187	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1188	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1189	pVCpu->hm.s.fForceTLBFlush = false;
1190	return;
1191	}
1192
1193
1194	/**
1195	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1196	*
1197	* @param pVM Pointer to the VM.
1198	* @param pVCpu Pointer to the VMCPU.
1199	* @remarks All references to "ASID" in this function pertains to "VPID" in
1200	* Intel's nomenclature. The reason is, to avoid confusion in compare
1201	* statements since the host-CPU copies are named "ASID".
1202	*
1203	* @remarks Called with interrupts disabled.
1204	*/
1205	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1206	{
1207	AssertPtr(pVM);
1208	AssertPtr(pVCpu);
1209	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1210	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1211	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1212
1213	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1214	AssertPtr(pCpu);
1215
1216	/*
1217	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1218	* This can happen both for start & resume due to long jumps back to ring-3.
1219	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1220	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1221	*/
1222	bool fNewASID = false;
1223	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1224	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1225	{
1226	pVCpu->hm.s.fForceTLBFlush = true;
1227	fNewASID = true;
1228	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1229	}
1230
1231	/*
1232	* Check for explicit TLB shootdowns.
1233	*/
1234	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1235	{
1236	pVCpu->hm.s.fForceTLBFlush = true;
1237	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1238	}
1239
1240	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1241	if (pVCpu->hm.s.fForceTLBFlush)
1242	{
1243	if (fNewASID)
1244	{
1245	++pCpu->uCurrentAsid;
1246	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1247	{
1248	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1249	pCpu->cTlbFlushes++;
1250	pCpu->fFlushAsidBeforeUse = true;
1251	}
1252
1253	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1254	if (pCpu->fFlushAsidBeforeUse)
1255	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1256	}
1257	else
1258	{
1259	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1260	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1261	else
1262	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1263	}
1264
1265	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1266	pVCpu->hm.s.fForceTLBFlush = false;
1267	}
1268	else
1269	{
1270	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1271	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1272	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1273	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1274
1275	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1276	* not be executed. See hmQueueInvlPage() where it is commented
1277	* out. Support individual entry flushing someday. */
1278	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1279	{
1280	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1281
1282	/*
1283	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1284	* as supported by the CPU.
1285	*/
1286	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1287	{
1288	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1289	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1290	}
1291	else
1292	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1293	}
1294	else
1295	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1296	}
1297	pVCpu->hm.s.TlbShootdown.cPages = 0;
1298	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1299
1300	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1301	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1302	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1304	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1305	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1306
1307	/* Update VMCS with the VPID. */
1308	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1309	AssertRC(rc);
1310	}
1311
1312
1313	/**
1314	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1315	*
1316	* @returns VBox status code.
1317	* @param pVM Pointer to the VM.
1318	* @param pVCpu Pointer to the VMCPU.
1319	*
1320	* @remarks Called with interrupts disabled.
1321	*/
1322	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1323	{
1324	AssertPtr(pVM);
1325	AssertPtr(pVCpu);
1326	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1327	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1328
1329	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1330	AssertPtr(pCpu);
1331
1332	/*
1333	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1334	* This can happen both for start & resume due to long jumps back to ring-3.
1335	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1336	*/
1337	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1338	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1339	{
1340	pVCpu->hm.s.fForceTLBFlush = true;
1341	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1342	}
1343
1344	/* Check for explicit TLB shootdown flushes. */
1345	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1346	{
1347	pVCpu->hm.s.fForceTLBFlush = true;
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1349	}
1350
1351	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1352	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1353
1354	if (pVCpu->hm.s.fForceTLBFlush)
1355	{
1356	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1357	pVCpu->hm.s.fForceTLBFlush = false;
1358	}
1359	else
1360	{
1361	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1362	* not be executed. See hmQueueInvlPage() where it is commented
1363	* out. Support individual entry flushing someday. */
1364	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1365	{
1366	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1368	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1369	}
1370	else
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1372	}
1373
1374	pVCpu->hm.s.TlbShootdown.cPages = 0;
1375	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1376	}
1377
1378
1379	/**
1380	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1381	*
1382	* @returns VBox status code.
1383	* @param pVM Pointer to the VM.
1384	* @param pVCpu Pointer to the VMCPU.
1385	*
1386	* @remarks Called with interrupts disabled.
1387	*/
1388	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1389	{
1390	AssertPtr(pVM);
1391	AssertPtr(pVCpu);
1392	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1393	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1394
1395	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1396
1397	/*
1398	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1399	* This can happen both for start & resume due to long jumps back to ring-3.
1400	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1401	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1402	*/
1403	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1404	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1405	{
1406	pVCpu->hm.s.fForceTLBFlush = true;
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1408	}
1409
1410	/* Check for explicit TLB shootdown flushes. */
1411	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1412	{
1413	/*
1414	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1415	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1416	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1417	*/
1418	pVCpu->hm.s.fForceTLBFlush = true;
1419	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1420	}
1421
1422	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1423	if (pVCpu->hm.s.fForceTLBFlush)
1424	{
1425	++pCpu->uCurrentAsid;
1426	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1427	{
1428	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1429	pCpu->fFlushAsidBeforeUse = true;
1430	pCpu->cTlbFlushes++;
1431	}
1432
1433	pVCpu->hm.s.fForceTLBFlush = false;
1434	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1435	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1436	if (pCpu->fFlushAsidBeforeUse)
1437	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1438	}
1439	else
1440	{
1441	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1442	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1443	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1444	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1445
1446	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1447	* not be executed. See hmQueueInvlPage() where it is commented
1448	* out. Support individual entry flushing someday. */
1449	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1450	{
1451	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1452	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1453	{
1454	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1455	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1456	}
1457	else
1458	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1459	}
1460	else
1461	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1462	}
1463
1464	pVCpu->hm.s.TlbShootdown.cPages = 0;
1465	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1466
1467	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1468	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1469	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1471	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1472	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1473
1474	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1475	AssertRC(rc);
1476	}
1477
1478
1479	/**
1480	* Flushes the guest TLB entry based on CPU capabilities.
1481	*
1482	* @param pVCpu Pointer to the VMCPU.
1483	*/
1484	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1485	{
1486	PVM pVM = pVCpu->CTX_SUFF(pVM);
1487	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1488	{
1489	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1491	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1492	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1493	default:
1494	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1495	break;
1496	}
1497	}
1498
1499
1500	/**
1501	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1502	* TLB entries from the host TLB before VM-entry.
1503	*
1504	* @returns VBox status code.
1505	* @param pVM Pointer to the VM.
1506	*/
1507	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1508	{
1509	/*
1510	* Determine optimal flush type for nested paging.
1511	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1512	* guest execution (see hmR3InitFinalizeR0()).
1513	*/
1514	if (pVM->hm.s.fNestedPaging)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1517	{
1518	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1520	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1521	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1522	else
1523	{
1524	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1525	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1526	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1527	}
1528
1529	/* Make sure the write-back cacheable memory type for EPT is supported. */
1530	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1531	{
1532	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1533	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1534	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1535	}
1536	}
1537	else
1538	{
1539	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1540	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1541	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1542	}
1543	}
1544
1545	/*
1546	* Determine optimal flush type for VPID.
1547	*/
1548	if (pVM->hm.s.vmx.fVpid)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1551	{
1552	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1554	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1555	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1556	else
1557	{
1558	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1561	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1562	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1563	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1564	pVM->hm.s.vmx.fVpid = false;
1565	}
1566	}
1567	else
1568	{
1569	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1570	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1571	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1572	pVM->hm.s.vmx.fVpid = false;
1573	}
1574	}
1575
1576	/*
1577	* Setup the handler for flushing tagged-TLBs.
1578	*/
1579	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1581	else if (pVM->hm.s.fNestedPaging)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1583	else if (pVM->hm.s.vmx.fVpid)
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1585	else
1586	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1587	return VINF_SUCCESS;
1588	}
1589
1590
1591	/**
1592	* Sets up pin-based VM-execution controls in the VMCS.
1593	*
1594	* @returns VBox status code.
1595	* @param pVM Pointer to the VM.
1596	* @param pVCpu Pointer to the VMCPU.
1597	*/
1598	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1599	{
1600	AssertPtr(pVM);
1601	AssertPtr(pVCpu);
1602
1603	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1604	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1605
1606	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1607	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1608	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1609
1610	/* Enable the VMX preemption timer. */
1611	if (pVM->hm.s.vmx.fUsePreemptTimer)
1612	{
1613	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1614	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1615	}
1616
1617	if ((val & zap) != val)
1618	{
1619	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1620	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1621	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1622	}
1623
1624	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1625	AssertRCReturn(rc, rc);
1626
1627	/* Update VCPU with the currently set pin-based VM-execution controls. */
1628	pVCpu->hm.s.vmx.u32PinCtls = val;
1629	return rc;
1630	}
1631
1632
1633	/**
1634	* Sets up processor-based VM-execution controls in the VMCS.
1635	*
1636	* @returns VBox status code.
1637	* @param pVM Pointer to the VM.
1638	* @param pVMCPU Pointer to the VMCPU.
1639	*/
1640	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1641	{
1642	AssertPtr(pVM);
1643	AssertPtr(pVCpu);
1644
1645	int rc = VERR_INTERNAL_ERROR_5;
1646	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1647	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1648
1649	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1654	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1655	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1656
1657	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1658	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1659	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1660	{
1661	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1662	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1663	}
1664
1665	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1666	if (!pVM->hm.s.fNestedPaging)
1667	{
1668	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1669	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1670	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1671	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1672	}
1673
1674	/* Use TPR shadowing if supported by the CPU. */
1675	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1676	{
1677	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1678	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1679	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1680	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1681	AssertRCReturn(rc, rc);
1682
1683	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1684	/* CR8 writes causes a VM-exit based on TPR threshold. */
1685	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1686	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1687	}
1688	else
1689	{
1690	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1691	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1692	}
1693
1694	/* Use MSR-bitmaps if supported by the CPU. */
1695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1696	{
1697	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1698
1699	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1701	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1702	AssertRCReturn(rc, rc);
1703
1704	/*
1705	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1706	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1707	*/
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1716	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1717	}
1718
1719	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1720	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1721	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1722
1723	if ((val & zap) != val)
1724	{
1725	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1726	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1727	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1728	}
1729
1730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1731	AssertRCReturn(rc, rc);
1732
1733	/* Update VCPU with the currently set processor-based VM-execution controls. */
1734	pVCpu->hm.s.vmx.u32ProcCtls = val;
1735
1736	/*
1737	* Secondary processor-based VM-execution controls.
1738	*/
1739	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1740	{
1741	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1742	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1743
1744	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1745	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1746
1747	if (pVM->hm.s.fNestedPaging)
1748	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1749	else
1750	{
1751	/*
1752	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1753	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1754	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1755	*/
1756	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1757	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1758	}
1759
1760	if (pVM->hm.s.vmx.fVpid)
1761	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1762
1763	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1764	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1765
1766	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1767	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1768	* done dynamically. */
1769	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1770	{
1771	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1772	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1773	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1774	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1775	AssertRCReturn(rc, rc);
1776	}
1777
1778	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1779	{
1780	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1781	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1782	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1783	}
1784
1785	if ((val & zap) != val)
1786	{
1787	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1788	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1789	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1790	}
1791
1792	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1793	AssertRCReturn(rc, rc);
1794
1795	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1796	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1797	}
1798
1799	return VINF_SUCCESS;
1800	}
1801
1802
1803	/**
1804	* Sets up miscellaneous (everything other than Pin & Processor-based
1805	* VM-execution) control fields in the VMCS.
1806	*
1807	* @returns VBox status code.
1808	* @param pVM Pointer to the VM.
1809	* @param pVCpu Pointer to the VMCPU.
1810	*/
1811	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1812	{
1813	AssertPtr(pVM);
1814	AssertPtr(pVCpu);
1815
1816	int rc = VERR_GENERAL_FAILURE;
1817
1818	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1819	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1820
1821	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1822
1823	/*
1824	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1825	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1826	* We thus use the exception bitmap to control it rather than use both.
1827	*/
1828	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1829	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1830
1831	/** @todo Explore possibility of using IO-bitmaps. */
1832	/* All IO & IOIO instructions cause VM-exits. */
1833	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1834	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1835
1836	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1837	/* Setup MSR autoloading/autostoring. */
1838	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1839	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1840	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1841	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1843	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1844
1845	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1846	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1847	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1848	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1849	#else
1850	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1851	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1852	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1853	#endif
1854
1855	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1856	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1857
1858	/* Setup debug controls */
1859	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1860	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1861	AssertRCReturn(rc, rc);
1862	return rc;
1863	}
1864
1865
1866	/**
1867	* Sets up the initial exception bitmap in the VMCS based on static conditions
1868	* (i.e. conditions that cannot ever change at runtime).
1869	*
1870	* @returns VBox status code.
1871	* @param pVM Pointer to the VM.
1872	* @param pVCpu Pointer to the VMCPU.
1873	*/
1874	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1875	{
1876	AssertPtr(pVM);
1877	AssertPtr(pVCpu);
1878
1879	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1880
1881	uint32_t u32XcptBitmap = 0;
1882
1883	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1884	if (!pVM->hm.s.fNestedPaging)
1885	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1886
1887	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1888	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1889	AssertRCReturn(rc, rc);
1890	return rc;
1891	}
1892
1893
1894	/**
1895	* Sets up the initial guest-state mask. The guest-state mask is consulted
1896	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1897	* for the nested virtualization case (as it would cause a VM-exit).
1898	*
1899	* @param pVCpu Pointer to the VMCPU.
1900	*/
1901	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1902	{
1903	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1904	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1905	return VINF_SUCCESS;
1906	}
1907
1908
1909	/**
1910	* Does per-VM VT-x initialization.
1911	*
1912	* @returns VBox status code.
1913	* @param pVM Pointer to the VM.
1914	*/
1915	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1916	{
1917	LogFlowFunc(("pVM=%p\n", pVM));
1918
1919	int rc = hmR0VmxStructsAlloc(pVM);
1920	if (RT_FAILURE(rc))
1921	{
1922	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1923	return rc;
1924	}
1925
1926	return VINF_SUCCESS;
1927	}
1928
1929
1930	/**
1931	* Does per-VM VT-x termination.
1932	*
1933	* @returns VBox status code.
1934	* @param pVM Pointer to the VM.
1935	*/
1936	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1937	{
1938	LogFlowFunc(("pVM=%p\n", pVM));
1939
1940	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1941	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1942	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1943	#endif
1944	hmR0VmxStructsFree(pVM);
1945	return VINF_SUCCESS;
1946	}
1947
1948
1949	/**
1950	* Sets up the VM for execution under VT-x.
1951	* This function is only called once per-VM during initalization.
1952	*
1953	* @returns VBox status code.
1954	* @param pVM Pointer to the VM.
1955	*/
1956	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1957	{
1958	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1959	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1960
1961	LogFlowFunc(("pVM=%p\n", pVM));
1962
1963	/*
1964	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1965	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1966	*/
1967	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1968	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1969	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1970	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1971	{
1972	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1973	return VERR_INTERNAL_ERROR;
1974	}
1975
1976	/* Initialize these always, see hmR3InitFinalizeR0().*/
1977	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1978	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1979
1980	/* Setup the tagged-TLB flush handlers. */
1981	int rc = hmR0VmxSetupTaggedTlb(pVM);
1982	if (RT_FAILURE(rc))
1983	{
1984	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1985	return rc;
1986	}
1987
1988	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1989	{
1990	PVMCPU pVCpu = &pVM->aCpus[i];
1991	AssertPtr(pVCpu);
1992	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1993
1994	/* Set revision dword at the beginning of the VMCS structure. */
1995	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1996
1997	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1998	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1999	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2000	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2001
2002	/* Load this VMCS as the current VMCS. */
2003	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2004	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2005	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2006
2007	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2008	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2009	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2010
2011	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2012	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2013	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2014
2015	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2016	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2017	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2018
2019	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2020	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2021	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2022
2023	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2024	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2025	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2026
2027	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2028	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2029	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2030	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2031	#endif
2032
2033	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2034	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2035	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2036	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2037
2038	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2039	}
2040
2041	return VINF_SUCCESS;
2042	}
2043
2044
2045	/**
2046	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2047	* the VMCS.
2048	*
2049	* @returns VBox status code.
2050	* @param pVM Pointer to the VM.
2051	* @param pVCpu Pointer to the VMCPU.
2052	*/
2053	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2054	{
2055	RTCCUINTREG uReg = ASMGetCR0();
2056	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2057
2058	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2059	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2060	if (HMVMX_IS_64BIT_HOST_MODE())
2061	{
2062	uint64_t uRegCR3 = hmR0Get64bitCR3();
2063	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2064	}
2065	else
2066	#endif
2067	{
2068	uReg = ASMGetCR3();
2069	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2070	}
2071
2072	uReg = ASMGetCR4();
2073	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2074	AssertRCReturn(rc, rc);
2075	return rc;
2076	}
2077
2078
2079	/**
2080	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2081	* the host-state area in the VMCS.
2082	*
2083	* @returns VBox status code.
2084	* @param pVM Pointer to the VM.
2085	* @param pVCpu Pointer to the VMCPU.
2086	*/
2087	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2088	{
2089	int rc = VERR_INTERNAL_ERROR_5;
2090	RTSEL uSelCS = 0;
2091	RTSEL uSelSS = 0;
2092	RTSEL uSelDS = 0;
2093	RTSEL uSelES = 0;
2094	RTSEL uSelFS = 0;
2095	RTSEL uSelGS = 0;
2096	RTSEL uSelTR = 0;
2097
2098	/*
2099	* Host Selector registers.
2100	*/
2101	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2102	if (HMVMX_IS_64BIT_HOST_MODE())
2103	{
2104	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2105	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2106	}
2107	else
2108	{
2109	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2110	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2111	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2112	}
2113	#else
2114	uSelCS = ASMGetCS();
2115	uSelSS = ASMGetSS();
2116	#endif
2117
2118	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2119	uSelTR = ASMGetTR();
2120
2121	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2122	/** @todo Verify if we have any platform that actually run with DS or ES with
2123	* RPL != 0 in kernel space. */
2124	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2125	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2126	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2127	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2128	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2129	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2130	Assert(uSelCS != 0);
2131	Assert(uSelTR != 0);
2132
2133	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2134	#if 0
2135	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2136	Assert(uSelSS != 0);
2137	#endif
2138
2139	/* Write these host selector fields into the host-state area in the VMCS. */
2140	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2141	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2142	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2143	#if 0
2144	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2145	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2146	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2147	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2148	#endif
2149	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2150	AssertRCReturn(rc, rc);
2151
2152	/*
2153	* Host GDTR and IDTR.
2154	*/
2155	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2156	* be safe to -not- save and restore GDTR and IDTR in the assembly
2157	* code and just do it here and don't care if the limits are zapped on
2158	* VM-exit. */
2159	RTGDTR Gdtr;
2160	RT_ZERO(Gdtr);
2161	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2162	if (HMVMX_IS_64BIT_HOST_MODE())
2163	{
2164	X86XDTR64 Gdtr64;
2165	X86XDTR64 Idtr64;
2166	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2167	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2168	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2169	Gdtr.cbGdt = Gdtr64.cb;
2170	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2171	}
2172	else
2173	#endif
2174	{
2175	RTIDTR Idtr;
2176	ASMGetGDTR(&Gdtr);
2177	ASMGetIDTR(&Idtr);
2178	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2179	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2180	}
2181	AssertRCReturn(rc, rc);
2182
2183	/*
2184	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2185	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2186	*/
2187	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2188	{
2189	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2190	return VERR_VMX_INVALID_HOST_STATE;
2191	}
2192
2193	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2194	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2195	if (HMVMX_IS_64BIT_HOST_MODE())
2196	{
2197	/* We need the 64-bit TR base for hybrid darwin. */
2198	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2199	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2200	}
2201	else
2202	#endif
2203	{
2204	uintptr_t uTRBase;
2205	#if HC_ARCH_BITS == 64
2206	uTRBase = X86DESC64_BASE(pDesc);
2207	#else
2208	uTRBase = X86DESC_BASE(pDesc);
2209	#endif
2210	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2211	}
2212	AssertRCReturn(rc, rc);
2213
2214	/*
2215	* Host FS base and GS base.
2216	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2217	* would take care of the bases. In 64-bit, the MSRs come into play.
2218	*/
2219	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2220	if (HMVMX_IS_64BIT_HOST_MODE())
2221	{
2222	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2223	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2224	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2225	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2226	AssertRCReturn(rc, rc);
2227	}
2228	#endif
2229	return rc;
2230	}
2231
2232
2233	/**
2234	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2235	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2236	* the host after every successful VM exit.
2237	*
2238	* @returns VBox status code.
2239	* @param pVM Pointer to the VM.
2240	* @param pVCpu Pointer to the VMCPU.
2241	*/
2242	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2243	{
2244	AssertPtr(pVCpu);
2245	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2246
2247	int rc = VINF_SUCCESS;
2248	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2249	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2250	uint32_t cHostMsrs = 0;
2251	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2252
2253	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2254	{
2255	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2256	pHostMsr->u32Reserved = 0;
2257	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2258	if (CPUMIsGuestInLongMode(pVCpu))
2259	{
2260	/* Must match the EFER value in our 64 bits switcher. */
2261	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2262	}
2263	else
2264	# endif
2265	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2266	pHostMsr++; cHostMsrs++;
2267	}
2268
2269	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2270	if (HMVMX_IS_64BIT_HOST_MODE())
2271	{
2272	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2273	pHostMsr->u32Reserved = 0;
2274	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2275	pHostMsr++; cHostMsrs++;
2276	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2277	pHostMsr->u32Reserved = 0;
2278	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2279	pHostMsr++; cHostMsrs++;
2280	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2281	pHostMsr->u32Reserved = 0;
2282	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2283	pHostMsr++; cHostMsrs++;
2284	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2285	pHostMsr->u32Reserved = 0;
2286	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2287	pHostMsr++; cHostMsrs++;
2288	}
2289	# endif
2290
2291	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2292	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2293	{
2294	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2295	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2296	}
2297
2298	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2299	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2300
2301	/*
2302	* Host Sysenter MSRs.
2303	*/
2304	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2305	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2306	if (HMVMX_IS_64BIT_HOST_MODE())
2307	{
2308	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2309	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2310	}
2311	else
2312	{
2313	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2314	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2315	}
2316	# elif HC_ARCH_BITS == 32
2317	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2318	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2319	# else
2320	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2321	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2322	# endif
2323	AssertRCReturn(rc, rc);
2324
2325	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2326	* hmR0VmxSetupExitCtls() !! */
2327	return rc;
2328	}
2329
2330
2331	/**
2332	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2333	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2334	* controls".
2335	*
2336	* @returns VBox status code.
2337	* @param pVCpu Pointer to the VMCPU.
2338	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2339	* out-of-sync. Make sure to update the required fields
2340	* before using them.
2341	*
2342	* @remarks No-long-jump zone!!!
2343	*/
2344	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2345	{
2346	int rc = VINF_SUCCESS;
2347	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2348	{
2349	PVM pVM = pVCpu->CTX_SUFF(pVM);
2350	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2351	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2352
2353	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2354	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2355
2356	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2357	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2358	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2359	else
2360	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2361
2362	/*
2363	* The following should not be set (since we're not in SMM mode):
2364	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2365	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2366	*/
2367
2368	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2369	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2370	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2371
2372	if ((val & zap) != val)
2373	{
2374	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2375	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2376	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2377	}
2378
2379	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2380	AssertRCReturn(rc, rc);
2381
2382	/* Update VCPU with the currently set VM-exit controls. */
2383	pVCpu->hm.s.vmx.u32EntryCtls = val;
2384	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2385	}
2386	return rc;
2387	}
2388
2389
2390	/**
2391	* Sets up the VM-exit controls in the VMCS.
2392	*
2393	* @returns VBox status code.
2394	* @param pVM Pointer to the VM.
2395	* @param pVCpu Pointer to the VMCPU.
2396	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2397	* out-of-sync. Make sure to update the required fields
2398	* before using them.
2399	*
2400	* @remarks requires EFER.
2401	*/
2402	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2403	{
2404	int rc = VINF_SUCCESS;
2405	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2406	{
2407	PVM pVM = pVCpu->CTX_SUFF(pVM);
2408	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2409	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2410
2411	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2412	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2413
2414	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2415	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2416	if (HMVMX_IS_64BIT_HOST_MODE())
2417	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2418	else
2419	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2420	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2421	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2422	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2423	else
2424	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2425	#endif
2426
2427	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2428	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2429
2430	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2431	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2432	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2433	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2434	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2435
2436	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2437	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2438
2439	if ((val & zap) != val)
2440	{
2441	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2442	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2443	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2444	}
2445
2446	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2447	AssertRCReturn(rc, rc);
2448
2449	/* Update VCPU with the currently set VM-exit controls. */
2450	pVCpu->hm.s.vmx.u32ExitCtls = val;
2451	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2452	}
2453	return rc;
2454	}
2455
2456
2457	/**
2458	* Loads the guest APIC and related state.
2459	*
2460	* @returns VBox status code.
2461	* @param pVM Pointer to the VM.
2462	* @param pVCpu Pointer to the VMCPU.
2463	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2464	* out-of-sync. Make sure to update the required fields
2465	* before using them.
2466	*/
2467	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2468	{
2469	int rc = VINF_SUCCESS;
2470	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2471	{
2472	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2473	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2474	{
2475	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2476
2477	bool fPendingIntr = false;
2478	uint8_t u8GuestTpr = 0;
2479	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2480	AssertRCReturn(rc, rc);
2481
2482	/*
2483	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2484	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2485	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2486	* the interrupt when we VM-exit for other reasons.
2487	*/
2488	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2489	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2490	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2491	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2492
2493	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2494	AssertRCReturn(rc, rc);
2495
2496	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2497	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2498	{
2499	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2500	pMixedCtx->msrLSTAR = u8GuestTpr;
2501	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2502	{
2503	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2504	if (fPendingIntr)
2505	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2506	else
2507	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508	}
2509	}
2510	}
2511
2512	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2513	}
2514	return rc;
2515	}
2516
2517
2518	/**
2519	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2520	*
2521	* @returns
2522	* @param pVCpu Pointer to the VMCPU.
2523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2524	* out-of-sync. Make sure to update the required fields
2525	* before using them.
2526	*
2527	* @remarks No-long-jump zone!!!
2528	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2529	*/
2530	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2531	{
2532	/*
2533	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2534	* inhibit interrupts or clear any existing interrupt-inhibition.
2535	*/
2536	uint32_t uIntrState = 0;
2537	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2538	{
2539	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2540	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2541	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2542	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2543	{
2544	/*
2545	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2546	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2547	*/
2548	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2549	}
2550	else if (pMixedCtx->eflags.Bits.u1IF)
2551	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2552	else
2553	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2554	}
2555	return uIntrState;
2556	}
2557
2558
2559	/**
2560	* Loads the guest's interruptibility-state into the guest-state area in the
2561	* VMCS.
2562	*
2563	* @returns VBox status code.
2564	* @param pVCpu Pointer to the VMCPU.
2565	* @param uIntrState The interruptibility-state to set.
2566	*/
2567	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2568	{
2569	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2570	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2571	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2572	AssertRCReturn(rc, rc);
2573	return rc;
2574	}
2575
2576
2577	/**
2578	* Loads the guest's RIP into the guest-state area in the VMCS.
2579	*
2580	* @returns VBox status code.
2581	* @param pVCpu Pointer to the VMCPU.
2582	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2583	* out-of-sync. Make sure to update the required fields
2584	* before using them.
2585	*
2586	* @remarks No-long-jump zone!!!
2587	*/
2588	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2589	{
2590	int rc = VINF_SUCCESS;
2591	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2592	{
2593	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2594	AssertRCReturn(rc, rc);
2595	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2596	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2597	}
2598	return rc;
2599	}
2600
2601
2602	/**
2603	* Loads the guest's RSP into the guest-state area in the VMCS.
2604	*
2605	* @returns VBox status code.
2606	* @param pVCpu Pointer to the VMCPU.
2607	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2608	* out-of-sync. Make sure to update the required fields
2609	* before using them.
2610	*
2611	* @remarks No-long-jump zone!!!
2612	*/
2613	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2614	{
2615	int rc = VINF_SUCCESS;
2616	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2617	{
2618	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2619	AssertRCReturn(rc, rc);
2620	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2621	}
2622	return rc;
2623	}
2624
2625
2626	/**
2627	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2628	*
2629	* @returns VBox status code.
2630	* @param pVCpu Pointer to the VMCPU.
2631	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2632	* out-of-sync. Make sure to update the required fields
2633	* before using them.
2634	*
2635	* @remarks No-long-jump zone!!!
2636	*/
2637	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2638	{
2639	int rc = VINF_SUCCESS;
2640	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2641	{
2642	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2643	Let us assert it as such and use 32-bit VMWRITE. */
2644	Assert(!(pMixedCtx->rflags.u64 >> 32));
2645	X86EFLAGS uEFlags = pMixedCtx->eflags;
2646	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2647	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2648
2649	/*
2650	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2651	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2652	*/
2653	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2654	{
2655	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2656	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2657	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2658	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2659	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2660	}
2661
2662	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2663	AssertRCReturn(rc, rc);
2664
2665	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2666	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2667	}
2668	return rc;
2669	}
2670
2671
2672	/**
2673	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2674	*
2675	* @returns VBox status code.
2676	* @param pVCpu Pointer to the VMCPU.
2677	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2678	* out-of-sync. Make sure to update the required fields
2679	* before using them.
2680	*
2681	* @remarks No-long-jump zone!!!
2682	*/
2683	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2684	{
2685	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2686	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2687	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2688	return rc;
2689	}
2690
2691
2692	/**
2693	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2694	* in the VMCS.
2695	*
2696	* @returns VBox status code.
2697	* @param pVM Pointer to the VM.
2698	* @param pVCpu Pointer to the VMCPU.
2699	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2700	* out-of-sync. Make sure to update the required fields
2701	* before using them.
2702	*
2703	* @remarks No-long-jump zone!!!
2704	*/
2705	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2706	{
2707	int rc = VINF_SUCCESS;
2708	PVM pVM = pVCpu->CTX_SUFF(pVM);
2709
2710	/*
2711	* Guest CR0.
2712	* Guest FPU.
2713	*/
2714	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2715	{
2716	Assert(!(pCtx->cr0 >> 32));
2717	uint32_t u32GuestCR0 = pCtx->cr0;
2718
2719	/* The guest's view (read access) of its CR0 is unblemished. */
2720	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2721	AssertRCReturn(rc, rc);
2722	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2723
2724	/* Setup VT-x's view of the guest CR0. */
2725	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2726	if (pVM->hm.s.fNestedPaging)
2727	{
2728	if (CPUMIsGuestPagingEnabledEx(pCtx))
2729	{
2730	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2731	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2732	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2733	}
2734	else
2735	{
2736	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2737	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2738	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2739	}
2740
2741	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2742	AssertRCReturn(rc, rc);
2743	}
2744	else
2745	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2746
2747	/*
2748	* Guest FPU bits.
2749	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2750	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2751	*/
2752	u32GuestCR0 \|= X86_CR0_NE;
2753	bool fInterceptNM = false;
2754	if (CPUMIsGuestFPUStateActive(pVCpu))
2755	{
2756	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2757	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2758	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2759	}
2760	else
2761	{
2762	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2763	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2764	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2765	}
2766
2767	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2768	bool fInterceptMF = false;
2769	if (!(pCtx->cr0 & X86_CR0_NE))
2770	fInterceptMF = true;
2771
2772	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2773	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2774	{
2775	Assert(PDMVmmDevHeapIsEnabled(pVM));
2776	Assert(pVM->hm.s.vmx.pRealModeTSS);
2777	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2778	fInterceptNM = true;
2779	fInterceptMF = true;
2780	}
2781	else
2782	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2783
2784	if (fInterceptNM)
2785	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2786	else
2787	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2788
2789	if (fInterceptMF)
2790	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2791	else
2792	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2793
2794	/* Additional intercepts for debugging, define these yourself explicitly. */
2795	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2796	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2797	\| RT_BIT(X86_XCPT_DB)
2798	\| RT_BIT(X86_XCPT_DE)
2799	\| RT_BIT(X86_XCPT_NM)
2800	\| RT_BIT(X86_XCPT_UD)
2801	\| RT_BIT(X86_XCPT_NP)
2802	\| RT_BIT(X86_XCPT_SS)
2803	\| RT_BIT(X86_XCPT_GP)
2804	\| RT_BIT(X86_XCPT_PF)
2805	\| RT_BIT(X86_XCPT_MF);
2806	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2807	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2808	#endif
2809
2810	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2811
2812	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2813	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2814	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2815	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2816	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2817	else
2818	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2819
2820	u32GuestCR0 \|= uSetCR0;
2821	u32GuestCR0 &= uZapCR0;
2822	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2823
2824	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2825	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2826	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2827	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2828
2829	/*
2830	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2831	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2832	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2833	*/
2834	uint32_t u32CR0Mask = 0;
2835	u32CR0Mask = X86_CR0_PE
2836	\| X86_CR0_NE
2837	\| X86_CR0_WP
2838	\| X86_CR0_PG
2839	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2840	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2841	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2842	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2843	u32CR0Mask &= ~X86_CR0_PE;
2844	if (pVM->hm.s.fNestedPaging)
2845	u32CR0Mask &= ~X86_CR0_WP;
2846
2847	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2848	if (fInterceptNM)
2849	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2850	else
2851	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2852
2853	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2854	pVCpu->hm.s.vmx.cr0_mask = u32CR0Mask;
2855	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2856	AssertRCReturn(rc, rc);
2857
2858	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2859	}
2860
2861	/*
2862	* Guest CR2.
2863	* It's always loaded in the assembler code. Nothing to do here.
2864	*/
2865
2866	/*
2867	* Guest CR3.
2868	*/
2869	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2870	{
2871	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2872	if (pVM->hm.s.fNestedPaging)
2873	{
2874	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2875
2876	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2877	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2878	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2879	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2880
2881	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2882	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2883	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2884
2885	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2886	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2887	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2888	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2889
2890	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2891	AssertRCReturn(rc, rc);
2892	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2893
2894	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2895	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2896	{
2897	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2898	if (CPUMIsGuestInPAEModeEx(pCtx))
2899	{
2900	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2901	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2902	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2903	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2904	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2905	AssertRCReturn(rc, rc);
2906	}
2907
2908	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2909	have Unrestricted Execution to handle the guest when it's not using paging. */
2910	GCPhysGuestCR3 = pCtx->cr3;
2911	}
2912	else
2913	{
2914	/*
2915	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2916	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2917	* EPT takes care of translating it to host-physical addresses.
2918	*/
2919	RTGCPHYS GCPhys;
2920	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2921	Assert(PDMVmmDevHeapIsEnabled(pVM));
2922
2923	/* We obtain it here every time as the guest could have relocated this PCI region. */
2924	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2925	AssertRCReturn(rc, rc);
2926
2927	GCPhysGuestCR3 = GCPhys;
2928	}
2929
2930	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
2931	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2932	}
2933	else
2934	{
2935	/* Non-nested paging case, just use the hypervisor's CR3. */
2936	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2937
2938	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
2939	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
2940	}
2941	AssertRCReturn(rc, rc);
2942
2943	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2944	}
2945
2946	/*
2947	* Guest CR4.
2948	*/
2949	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2950	{
2951	Assert(!(pCtx->cr4 >> 32));
2952	uint32_t u32GuestCR4 = pCtx->cr4;
2953
2954	/* The guest's view of its CR4 is unblemished. */
2955	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2956	AssertRCReturn(rc, rc);
2957	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2958
2959	/* Setup VT-x's view of the guest CR4. */
2960	/*
2961	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2962	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2963	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2964	*/
2965	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2966	{
2967	Assert(pVM->hm.s.vmx.pRealModeTSS);
2968	Assert(PDMVmmDevHeapIsEnabled(pVM));
2969	u32GuestCR4 &= ~X86_CR4_VME;
2970	}
2971
2972	if (pVM->hm.s.fNestedPaging)
2973	{
2974	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2975	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2976	{
2977	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2978	u32GuestCR4 \|= X86_CR4_PSE;
2979	/* Our identity mapping is a 32 bits page directory. */
2980	u32GuestCR4 &= ~X86_CR4_PAE;
2981	}
2982	/* else use guest CR4.*/
2983	}
2984	else
2985	{
2986	/*
2987	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2988	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2989	*/
2990	switch (pVCpu->hm.s.enmShadowMode)
2991	{
2992	case PGMMODE_REAL: /* Real-mode. */
2993	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2994	case PGMMODE_32_BIT: /* 32-bit paging. */
2995	{
2996	u32GuestCR4 &= ~X86_CR4_PAE;
2997	break;
2998	}
2999
3000	case PGMMODE_PAE: /* PAE paging. */
3001	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3002	{
3003	u32GuestCR4 \|= X86_CR4_PAE;
3004	break;
3005	}
3006
3007	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3008	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3009	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3010	break;
3011	#endif
3012	default:
3013	AssertFailed();
3014	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3015	}
3016	}
3017
3018	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3019	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3020	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3021	u32GuestCR4 \|= uSetCR4;
3022	u32GuestCR4 &= uZapCR4;
3023
3024	/* Write VT-x's view of the guest CR4 into the VMCS. */
3025	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3026	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3027
3028	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3029	uint32_t u32CR4Mask = 0;
3030	u32CR4Mask = X86_CR4_VME
3031	\| X86_CR4_PAE
3032	\| X86_CR4_PGE
3033	\| X86_CR4_PSE
3034	\| X86_CR4_VMXE;
3035	pVCpu->hm.s.vmx.cr4_mask = u32CR4Mask;
3036	rc \|= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3037	AssertRCReturn(rc, rc);
3038
3039	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3040	}
3041	return rc;
3042	}
3043
3044
3045	/**
3046	* Loads the guest debug registers into the guest-state area in the VMCS.
3047	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3048	*
3049	* @returns VBox status code.
3050	* @param pVCpu Pointer to the VMCPU.
3051	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3052	* out-of-sync. Make sure to update the required fields
3053	* before using them.
3054	*
3055	* @remarks No-long-jump zone!!!
3056	*/
3057	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3058	{
3059	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3060	return VINF_SUCCESS;
3061
3062	#ifdef VBOX_STRICT
3063	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3064	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3065	{
3066	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3067	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3068	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3069	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3070	}
3071	#endif
3072
3073	int rc = VERR_INTERNAL_ERROR_5;
3074	PVM pVM = pVCpu->CTX_SUFF(pVM);
3075	bool fInterceptDB = false;
3076	bool fInterceptMovDRx = false;
3077	if (DBGFIsStepping(pVCpu))
3078	{
3079	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3080	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3081	{
3082	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3083	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3084	AssertRCReturn(rc, rc);
3085	Assert(fInterceptDB == false);
3086	}
3087	else
3088	fInterceptDB = true;
3089	}
3090
3091	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3092	{
3093	if (!CPUMIsHyperDebugStateActive(pVCpu))
3094	{
3095	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3096	AssertRC(rc);
3097	}
3098	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3099	fInterceptMovDRx = true;
3100	}
3101	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3102	{
3103	if (!CPUMIsGuestDebugStateActive(pVCpu))
3104	{
3105	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3106	AssertRC(rc);
3107	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3108	}
3109	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3110	Assert(fInterceptMovDRx == false);
3111	}
3112	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3113	{
3114	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3115	fInterceptMovDRx = true;
3116	}
3117
3118	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3119	if (fInterceptDB)
3120	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3121	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3122	{
3123	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3124	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3125	#endif
3126	}
3127
3128	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3129	if (fInterceptMovDRx)
3130	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3131	else
3132	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3133
3134	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3135	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3136
3137	/* The guest's view of its DR7 is unblemished. */
3138	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3139
3140	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3141	return rc;
3142	}
3143
3144
3145	#ifdef VBOX_STRICT
3146	/**
3147	* Strict function to validate segment registers.
3148	*
3149	* @remarks Requires CR0.
3150	*/
3151	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3152	{
3153	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3154	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3155	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3156
3157	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3158	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3159	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3160	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3161	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3162	{
3163	/* Protected mode checks */
3164	/* CS */
3165	Assert(pCtx->cs.Attr.n.u1Present);
3166	Assert(!(pCtx->cs.Attr.u & 0xf00));
3167	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3168	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3169	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3170	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3171	\|\| (pCtx->cs.Attr.n.u1Granularity));
3172	/* CS cannot be loaded with NULL in protected mode. */
3173	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3174	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3175	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3176	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3177	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3178	else
3179	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3180	/* SS */
3181	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3182	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3183	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3184	if ( !(pCtx->cr0 & X86_CR0_PE)
3185	\|\| pCtx->cs.Attr.n.u4Type == 3)
3186	{
3187	Assert(!pCtx->ss.Attr.n.u2Dpl);
3188	}
3189	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3190	{
3191	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3192	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3193	Assert(pCtx->ss.Attr.n.u1Present);
3194	Assert(!(pCtx->ss.Attr.u & 0xf00));
3195	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3196	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3197	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3198	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3199	\|\| (pCtx->ss.Attr.n.u1Granularity));
3200	}
3201	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3202	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3203	{
3204	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3205	Assert(pCtx->ds.Attr.n.u1Present);
3206	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3207	Assert(!(pCtx->ds.Attr.u & 0xf00));
3208	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3209	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3210	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3211	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3212	\|\| (pCtx->ds.Attr.n.u1Granularity));
3213	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3214	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3215	}
3216	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3217	{
3218	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3219	Assert(pCtx->es.Attr.n.u1Present);
3220	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3221	Assert(!(pCtx->es.Attr.u & 0xf00));
3222	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3223	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3224	\|\| !(pCtx->es.Attr.n.u1Granularity));
3225	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3226	\|\| (pCtx->es.Attr.n.u1Granularity));
3227	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3228	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3229	}
3230	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3231	{
3232	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3233	Assert(pCtx->fs.Attr.n.u1Present);
3234	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3235	Assert(!(pCtx->fs.Attr.u & 0xf00));
3236	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3237	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3238	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3239	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3240	\|\| (pCtx->fs.Attr.n.u1Granularity));
3241	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3242	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3243	}
3244	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3245	{
3246	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3247	Assert(pCtx->gs.Attr.n.u1Present);
3248	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3249	Assert(!(pCtx->gs.Attr.u & 0xf00));
3250	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3251	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3252	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3253	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3254	\|\| (pCtx->gs.Attr.n.u1Granularity));
3255	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3256	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3257	}
3258	/* 64-bit capable CPUs. */
3259	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3260	Assert(!(pCtx->cs.u64Base >> 32));
3261	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3262	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3263	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3264	# endif
3265	}
3266	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3267	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3268	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3269	{
3270	/* Real and v86 mode checks. */
3271	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3272	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3273	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3274	{
3275	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3276	}
3277	else
3278	{
3279	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3280	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3281	}
3282
3283	/* CS */
3284	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3285	Assert(pCtx->cs.u32Limit == 0xffff);
3286	Assert(u32CSAttr == 0xf3);
3287	/* SS */
3288	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3289	Assert(pCtx->ss.u32Limit == 0xffff);
3290	Assert(u32SSAttr == 0xf3);
3291	/* DS */
3292	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3293	Assert(pCtx->ds.u32Limit == 0xffff);
3294	Assert(u32DSAttr == 0xf3);
3295	/* ES */
3296	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3297	Assert(pCtx->es.u32Limit == 0xffff);
3298	Assert(u32ESAttr == 0xf3);
3299	/* FS */
3300	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3301	Assert(pCtx->fs.u32Limit == 0xffff);
3302	Assert(u32FSAttr == 0xf3);
3303	/* GS */
3304	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3305	Assert(pCtx->gs.u32Limit == 0xffff);
3306	Assert(u32GSAttr == 0xf3);
3307	/* 64-bit capable CPUs. */
3308	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3309	Assert(!(pCtx->cs.u64Base >> 32));
3310	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3311	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3312	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3313	# endif
3314	}
3315	}
3316	#endif /* VBOX_STRICT */
3317
3318
3319	/**
3320	* Writes a guest segment register into the guest-state area in the VMCS.
3321	*
3322	* @returns VBox status code.
3323	* @param pVCpu Pointer to the VMCPU.
3324	* @param idxSel Index of the selector in the VMCS.
3325	* @param idxLimit Index of the segment limit in the VMCS.
3326	* @param idxBase Index of the segment base in the VMCS.
3327	* @param idxAccess Index of the access rights of the segment in the VMCS.
3328	* @param pSelReg Pointer to the segment selector.
3329	* @param pCtx Pointer to the guest-CPU context.
3330	*
3331	* @remarks No-long-jump zone!!!
3332	*/
3333	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3334	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3335	{
3336	int rc;
3337	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3338	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3339	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3340	AssertRCReturn(rc, rc);
3341
3342	uint32_t u32Access = pSelReg->Attr.u;
3343	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3344	{
3345	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3346	u32Access = 0xf3;
3347	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3348	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3349	}
3350	else
3351	{
3352	/*
3353	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3354	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3355	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3356	* loaded in protected-mode have their attribute as 0.
3357	*/
3358	if (!u32Access)
3359	u32Access = HMVMX_SEL_UNUSABLE;
3360	}
3361
3362	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3363	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3364	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3365
3366	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3367	AssertRCReturn(rc, rc);
3368	return rc;
3369	}
3370
3371
3372	/**
3373	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3374	* into the guest-state area in the VMCS.
3375	*
3376	* @returns VBox status code.
3377	* @param pVM Pointer to the VM.
3378	* @param pVCPU Pointer to the VMCPU.
3379	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3380	* out-of-sync. Make sure to update the required fields
3381	* before using them.
3382	*
3383	* @remarks Requires CR0 (strict builds validation).
3384	* @remarks No-long-jump zone!!!
3385	*/
3386	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3387	{
3388	int rc = VERR_INTERNAL_ERROR_5;
3389	PVM pVM = pVCpu->CTX_SUFF(pVM);
3390
3391	/*
3392	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3393	*/
3394	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3395	{
3396	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3397	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3398	{
3399	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3400	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3401	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3402	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3403	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3404	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3405	}
3406
3407	#ifdef VBOX_WITH_REM
3408	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3409	{
3410	Assert(pVM->hm.s.vmx.pRealModeTSS);
3411	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3412	if ( pVCpu->hm.s.vmx.fWasInRealMode
3413	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3414	{
3415	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3416	in real-mode (e.g. OpenBSD 4.0) */
3417	REMFlushTBs(pVM);
3418	Log(("Load: Switch to protected mode detected!\n"));
3419	pVCpu->hm.s.vmx.fWasInRealMode = false;
3420	}
3421	}
3422	#endif
3423	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3424	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3425	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3426	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3427	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3428	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3429	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3430	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3431	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3432	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3433	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3434	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3435	AssertRCReturn(rc, rc);
3436
3437	#ifdef VBOX_STRICT
3438	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3439	#endif
3440	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3441	}
3442
3443	/*
3444	* Guest TR.
3445	*/
3446	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3447	{
3448	/*
3449	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3450	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3451	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3452	*/
3453	uint16_t u16Sel = 0;
3454	uint32_t u32Limit = 0;
3455	uint64_t u64Base = 0;
3456	uint32_t u32AccessRights = 0;
3457
3458	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3459	{
3460	u16Sel = pMixedCtx->tr.Sel;
3461	u32Limit = pMixedCtx->tr.u32Limit;
3462	u64Base = pMixedCtx->tr.u64Base;
3463	u32AccessRights = pMixedCtx->tr.Attr.u;
3464	}
3465	else
3466	{
3467	Assert(pVM->hm.s.vmx.pRealModeTSS);
3468	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3469
3470	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3471	RTGCPHYS GCPhys;
3472	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3473	AssertRCReturn(rc, rc);
3474
3475	X86DESCATTR DescAttr;
3476	DescAttr.u = 0;
3477	DescAttr.n.u1Present = 1;
3478	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3479
3480	u16Sel = 0;
3481	u32Limit = HM_VTX_TSS_SIZE;
3482	u64Base = GCPhys; /* in real-mode phys = virt. */
3483	u32AccessRights = DescAttr.u;
3484	}
3485
3486	/* Validate. */
3487	Assert(!(u16Sel & RT_BIT(2)));
3488	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3489	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3490	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3491	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3492	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3493	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3494	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3495	Assert( (u32Limit & 0xfff) == 0xfff
3496	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3497	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3498	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3499
3500	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3501	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3502	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3503	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3504	AssertRCReturn(rc, rc);
3505
3506	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3507	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3508	}
3509
3510	/*
3511	* Guest GDTR.
3512	*/
3513	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3514	{
3515	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3516	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3517	AssertRCReturn(rc, rc);
3518
3519	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3520	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3521	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3522	}
3523
3524	/*
3525	* Guest LDTR.
3526	*/
3527	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3528	{
3529	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3530	uint32_t u32Access = 0;
3531	if (!pMixedCtx->ldtr.Attr.u)
3532	u32Access = HMVMX_SEL_UNUSABLE;
3533	else
3534	u32Access = pMixedCtx->ldtr.Attr.u;
3535
3536	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3537	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3538	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3539	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3540	AssertRCReturn(rc, rc);
3541
3542	/* Validate. */
3543	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3544	{
3545	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3546	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3547	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3548	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3549	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3550	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3551	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3552	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3553	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3554	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3555	}
3556
3557	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3558	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3559	}
3560
3561	/*
3562	* Guest IDTR.
3563	*/
3564	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3565	{
3566	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3567	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3568	AssertRCReturn(rc, rc);
3569
3570	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3571	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3572	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3573	}
3574
3575	return VINF_SUCCESS;
3576	}
3577
3578
3579	/**
3580	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3581	* areas. These MSRs will automatically be loaded to the host CPU on every
3582	* successful VM entry and stored from the host CPU on every successful VM exit.
3583	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3584	*
3585	* @returns VBox status code.
3586	* @param pVCpu Pointer to the VMCPU.
3587	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588	* out-of-sync. Make sure to update the required fields
3589	* before using them.
3590	*
3591	* @remarks No-long-jump zone!!!
3592	*/
3593	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3594	{
3595	AssertPtr(pVCpu);
3596	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3597
3598	/*
3599	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3600	*/
3601	int rc = VINF_SUCCESS;
3602	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3603	{
3604	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3605	PVM pVM = pVCpu->CTX_SUFF(pVM);
3606	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3607	uint32_t cGuestMsrs = 0;
3608
3609	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3610	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3611	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3612	if (fSupportsNX \|\| fSupportsLongMode)
3613	{
3614	/** @todo support save IA32_EFER, i.e.
3615	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3616	* guest EFER need not be part of the VM-entry MSR-load area. */
3617	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3618	pGuestMsr->u32Reserved = 0;
3619	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3620	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3621	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3622	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3623	pGuestMsr++; cGuestMsrs++;
3624	if (fSupportsLongMode)
3625	{
3626	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3627	pGuestMsr->u32Reserved = 0;
3628	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3629	pGuestMsr++; cGuestMsrs++;
3630	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3631	pGuestMsr->u32Reserved = 0;
3632	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3633	pGuestMsr++; cGuestMsrs++;
3634	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3635	pGuestMsr->u32Reserved = 0;
3636	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3637	pGuestMsr++; cGuestMsrs++;
3638	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3639	pGuestMsr->u32Reserved = 0;
3640	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3641	pGuestMsr++; cGuestMsrs++;
3642	}
3643	}
3644
3645	/*
3646	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3647	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3648	*/
3649	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3650	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3651	{
3652	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3653	pGuestMsr->u32Reserved = 0;
3654	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3655	AssertRCReturn(rc, rc);
3656	pGuestMsr++; cGuestMsrs++;
3657	}
3658
3659	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3660	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3661	{
3662	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3663	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3664	}
3665
3666	/* Update the VCPU's copy of the guest MSR count. */
3667	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3668	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3669	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3670	AssertRCReturn(rc, rc);
3671	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3672
3673	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3674	}
3675
3676	/*
3677	* Guest Sysenter MSRs.
3678	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3679	* VM-exits on WRMSRs for these MSRs.
3680	*/
3681	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3682	{
3683	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3684	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3685	}
3686	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3687	{
3688	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3689	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3690	}
3691	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3692	{
3693	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3694	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3695	}
3696	AssertRCReturn(rc, rc);
3697
3698	return rc;
3699	}
3700
3701
3702	/**
3703	* Loads the guest activity state into the guest-state area in the VMCS.
3704	*
3705	* @returns VBox status code.
3706	* @param pVCpu Pointer to the VMCPU.
3707	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3708	* out-of-sync. Make sure to update the required fields
3709	* before using them.
3710	*
3711	* @remarks No-long-jump zone!!!
3712	*/
3713	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3714	{
3715	/** @todo See if we can make use of other states, e.g.
3716	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3717	int rc = VINF_SUCCESS;
3718	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3719	{
3720	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3721	AssertRCReturn(rc, rc);
3722	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3723	}
3724	return rc;
3725	}
3726
3727
3728	/**
3729	* Sets up the appropriate function to run guest code.
3730	*
3731	* @returns VBox status code.
3732	* @param pVCpu Pointer to the VMCPU.
3733	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3734	* out-of-sync. Make sure to update the required fields
3735	* before using them.
3736	*
3737	* @remarks No-long-jump zone!!!
3738	*/
3739	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3740	{
3741	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3742	{
3743	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3744	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3745	#endif
3746	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3747	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3748	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3749	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3750	#else
3751	/* 64-bit host or hybrid host. */
3752	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3753	#endif
3754	}
3755	else
3756	{
3757	/* Guest is not in long mode, use the 32-bit handler. */
3758	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3759	}
3760	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3761	return VINF_SUCCESS;
3762	}
3763
3764
3765	/**
3766	* Wrapper for running the guest code in VT-x.
3767	*
3768	* @returns VBox strict status code.
3769	* @param pVM Pointer to the VM.
3770	* @param pVCpu Pointer to the VMCPU.
3771	* @param pCtx Pointer to the guest-CPU context.
3772	*
3773	* @remarks No-long-jump zone!!!
3774	*/
3775	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3776	{
3777	/*
3778	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3779	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3780	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3781	*/
3782	#ifdef VBOX_WITH_KERNEL_USING_XMM
3783	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3784	#else
3785	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3786	#endif
3787	}
3788
3789
3790	/**
3791	* Report world-switch error and dump some useful debug info.
3792	*
3793	* @param pVM Pointer to the VM.
3794	* @param pVCpu Pointer to the VMCPU.
3795	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3796	* @param pCtx Pointer to the guest-CPU context.
3797	* @param pVmxTransient Pointer to the VMX transient structure (only
3798	* exitReason updated).
3799	*/
3800	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3801	{
3802	Assert(pVM);
3803	Assert(pVCpu);
3804	Assert(pCtx);
3805	Assert(pVmxTransient);
3806	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3807
3808	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3809	switch (rcVMRun)
3810	{
3811	case VERR_VMX_INVALID_VMXON_PTR:
3812	AssertFailed();
3813	break;
3814	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3815	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3816	{
3817	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3818	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3819	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3820	AssertRC(rc);
3821
3822	#ifdef VBOX_STRICT
3823	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3824	pVmxTransient->uExitReason));
3825	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3826	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3827	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3828	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3829	else
3830	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3831
3832	/* VMX control bits. */
3833	uint32_t u32Val;
3834	uint64_t u64Val;
3835	HMVMXHCUINTREG uHCReg;
3836	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3837	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3838	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3839	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3840	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3841	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3842	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3843	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3844	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3845	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3846	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3847	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3848	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3849	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3850	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3851	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3852	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3853	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3854	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3855	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3856	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3857	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3858	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3859	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3860	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3861	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3862	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3863	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3864	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3865	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3866	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3867	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3868	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3869	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3870	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3871	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3872	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3873	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3874	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3875	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3876	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3877	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3878
3879	/* Guest bits. */
3880	RTGCUINTREG uGCReg;
3881	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3882	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3883	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3884	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3885	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3886	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3887	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3888	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3889
3890	/* Host bits. */
3891	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3892	Log(("Host CR0 %#RHr\n", uHCReg));
3893	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3894	Log(("Host CR3 %#RHr\n", uHCReg));
3895	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3896	Log(("Host CR4 %#RHr\n", uHCReg));
3897
3898	RTGDTR HostGdtr;
3899	PCX86DESCHC pDesc;
3900	ASMGetGDTR(&HostGdtr);
3901	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3902	Log(("Host CS %#08x\n", u32Val));
3903	if (u32Val < HostGdtr.cbGdt)
3904	{
3905	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3906	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3907	}
3908
3909	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3910	Log(("Host DS %#08x\n", u32Val));
3911	if (u32Val < HostGdtr.cbGdt)
3912	{
3913	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3914	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3915	}
3916
3917	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3918	Log(("Host ES %#08x\n", u32Val));
3919	if (u32Val < HostGdtr.cbGdt)
3920	{
3921	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3922	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3923	}
3924
3925	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3926	Log(("Host FS %#08x\n", u32Val));
3927	if (u32Val < HostGdtr.cbGdt)
3928	{
3929	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3930	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3931	}
3932
3933	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3934	Log(("Host GS %#08x\n", u32Val));
3935	if (u32Val < HostGdtr.cbGdt)
3936	{
3937	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3938	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3939	}
3940
3941	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3942	Log(("Host SS %#08x\n", u32Val));
3943	if (u32Val < HostGdtr.cbGdt)
3944	{
3945	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3946	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3947	}
3948
3949	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3950	Log(("Host TR %#08x\n", u32Val));
3951	if (u32Val < HostGdtr.cbGdt)
3952	{
3953	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3954	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3955	}
3956
3957	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3958	Log(("Host TR Base %#RHv\n", uHCReg));
3959	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3960	Log(("Host GDTR Base %#RHv\n", uHCReg));
3961	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3962	Log(("Host IDTR Base %#RHv\n", uHCReg));
3963	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3964	Log(("Host SYSENTER CS %#08x\n", u32Val));
3965	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3966	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3967	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3968	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3969	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3970	Log(("Host RSP %#RHv\n", uHCReg));
3971	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3972	Log(("Host RIP %#RHv\n", uHCReg));
3973	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3974	if (HMVMX_IS_64BIT_HOST_MODE())
3975	{
3976	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3977	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3978	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3979	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3980	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3981	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3982	}
3983	# endif
3984	#endif /* VBOX_STRICT */
3985	break;
3986	}
3987
3988	default:
3989	/* Impossible */
3990	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3991	break;
3992	}
3993	NOREF(pVM);
3994	}
3995
3996
3997	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3998	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3999	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4000	#endif
4001	#ifdef VBOX_STRICT
4002	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4003	{
4004	switch (idxField)
4005	{
4006	case VMX_VMCS_GUEST_RIP:
4007	case VMX_VMCS_GUEST_RSP:
4008	case VMX_VMCS_GUEST_DR7:
4009	case VMX_VMCS_GUEST_SYSENTER_EIP:
4010	case VMX_VMCS_GUEST_SYSENTER_ESP:
4011	case VMX_VMCS_GUEST_GDTR_BASE:
4012	case VMX_VMCS_GUEST_IDTR_BASE:
4013	case VMX_VMCS_GUEST_CS_BASE:
4014	case VMX_VMCS_GUEST_DS_BASE:
4015	case VMX_VMCS_GUEST_ES_BASE:
4016	case VMX_VMCS_GUEST_FS_BASE:
4017	case VMX_VMCS_GUEST_GS_BASE:
4018	case VMX_VMCS_GUEST_SS_BASE:
4019	case VMX_VMCS_GUEST_LDTR_BASE:
4020	case VMX_VMCS_GUEST_TR_BASE:
4021	case VMX_VMCS_GUEST_CR3:
4022	return true;
4023	}
4024	return false;
4025	}
4026
4027	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4028	{
4029	switch (idxField)
4030	{
4031	/* Read-only fields. */
4032	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4033	return true;
4034	}
4035	/* Remaining readable fields should also be writable. */
4036	return hmR0VmxIsValidWriteField(idxField);
4037	}
4038	#endif /* VBOX_STRICT */
4039
4040	/**
4041	* Executes the specified handler in 64-bit mode.
4042	*
4043	* @returns VBox status code.
4044	* @param pVM Pointer to the VM.
4045	* @param pVCpu Pointer to the VMCPU.
4046	* @param pCtx Pointer to the guest CPU context.
4047	* @param enmOp The operation to perform.
4048	* @param cbParam Number of parameters.
4049	* @param paParam Array of 32-bit parameters.
4050	*/
4051	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4052	uint32_t *paParam)
4053	{
4054	int rc, rc2;
4055	PHMGLOBLCPUINFO pCpu;
4056	RTHCPHYS HCPhysCpuPage;
4057	RTCCUINTREG uOldEFlags;
4058
4059	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4060	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4061	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4062	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4063
4064	#ifdef VBOX_STRICT
4065	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4066	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4067
4068	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4069	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4070	#endif
4071
4072	/* Disable interrupts. */
4073	uOldEFlags = ASMIntDisableFlags();
4074
4075	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4076	RTCPUID idHostCpu = RTMpCpuId();
4077	CPUMR0SetLApic(pVM, idHostCpu);
4078	#endif
4079
4080	pCpu = HMR0GetCurrentCpu();
4081	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4082
4083	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4084	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4085
4086	/* Leave VMX Root Mode. */
4087	VMXDisable();
4088
4089	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4090
4091	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4092	CPUMSetHyperEIP(pVCpu, enmOp);
4093	for (int i = (int)cbParam - 1; i >= 0; i--)
4094	CPUMPushHyper(pVCpu, paParam[i]);
4095
4096	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4097
4098	/* Call the switcher. */
4099	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4100	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4101
4102	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4103	/* Make sure the VMX instructions don't cause #UD faults. */
4104	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4105
4106	/* Re-enter VMX Root Mode */
4107	rc2 = VMXEnable(HCPhysCpuPage);
4108	if (RT_FAILURE(rc2))
4109	{
4110	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4111	ASMSetFlags(uOldEFlags);
4112	return rc2;
4113	}
4114
4115	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4116	AssertRC(rc2);
4117	Assert(!(ASMGetFlags() & X86_EFL_IF));
4118	ASMSetFlags(uOldEFlags);
4119	return rc;
4120	}
4121
4122
4123	/**
4124	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4125	* supporting 64-bit guests.
4126	*
4127	* @returns VBox status code.
4128	* @param fResume Whether to VMLAUNCH or VMRESUME.
4129	* @param pCtx Pointer to the guest-CPU context.
4130	* @param pCache Pointer to the VMCS cache.
4131	* @param pVM Pointer to the VM.
4132	* @param pVCpu Pointer to the VMCPU.
4133	*/
4134	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4135	{
4136	uint32_t aParam[6];
4137	PHMGLOBLCPUINFO pCpu = NULL;
4138	RTHCPHYS HCPhysCpuPage = 0;
4139	int rc = VERR_INTERNAL_ERROR_5;
4140
4141	pCpu = HMR0GetCurrentCpu();
4142	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4143
4144	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4145	pCache->uPos = 1;
4146	pCache->interPD = PGMGetInterPaeCR3(pVM);
4147	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4148	#endif
4149
4150	#ifdef VBOX_STRICT
4151	pCache->TestIn.HCPhysCpuPage = 0;
4152	pCache->TestIn.HCPhysVmcs = 0;
4153	pCache->TestIn.pCache = 0;
4154	pCache->TestOut.HCPhysVmcs = 0;
4155	pCache->TestOut.pCache = 0;
4156	pCache->TestOut.pCtx = 0;
4157	pCache->TestOut.eflags = 0;
4158	#endif
4159
4160	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4161	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4162	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4163	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4164	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4165	aParam[5] = 0;
4166
4167	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4168	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4169	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4170	#endif
4171	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4172
4173	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4174	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4175	Assert(pCtx->dr[4] == 10);
4176	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4177	#endif
4178
4179	#ifdef VBOX_STRICT
4180	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4181	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4182	pVCpu->hm.s.vmx.HCPhysVmcs));
4183	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4184	pCache->TestOut.HCPhysVmcs));
4185	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4186	pCache->TestOut.pCache));
4187	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4188	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4189	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4190	pCache->TestOut.pCtx));
4191	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4192	#endif
4193	return rc;
4194	}
4195
4196
4197	/**
4198	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4199	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4200	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4201	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4202	*
4203	* @returns VBox status code.
4204	* @param pVM Pointer to the VM.
4205	* @param pVCpu Pointer to the VMCPU.
4206	*/
4207	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4208	{
4209	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4210	{ \
4211	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4212	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4213	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4214	++cReadFields; \
4215	}
4216
4217	AssertPtr(pVM);
4218	AssertPtr(pVCpu);
4219	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4220	uint32_t cReadFields = 0;
4221
4222	/* Guest-natural selector base fields */
4223	#if 0
4224	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4225	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4226	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4227	#endif
4228	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4229	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4230	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4231	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4232	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4239	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4241	#if 0
4242	/* Unused natural width guest-state fields. */
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4245	#endif
4246	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4248
4249	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4250	#if 0
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4252	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4257	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4260	#endif
4261
4262	/* Natural width guest-state fields. */
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4264	#if 0
4265	/* Currently unused field. */
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4267	#endif
4268
4269	if (pVM->hm.s.fNestedPaging)
4270	{
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4272	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4273	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4274	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4275	}
4276	else
4277	{
4278	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4279	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4280	}
4281
4282	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4283	return VINF_SUCCESS;
4284	}
4285
4286
4287	/**
4288	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4289	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4290	* darwin, running 64-bit guests).
4291	*
4292	* @returns VBox status code.
4293	* @param pVCpu Pointer to the VMCPU.
4294	* @param idxField The VMCS field encoding.
4295	* @param u64Val 16, 32 or 64 bits value.
4296	*/
4297	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4298	{
4299	int rc;
4300	switch (idxField)
4301	{
4302	/*
4303	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4304	*/
4305	/* 64-bit Control fields. */
4306	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4307	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4308	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4309	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4310	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4311	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4312	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4313	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4314	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4315	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4316	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4317	case VMX_VMCS64_CTRL_EPTP_FULL:
4318	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4319	/* 64-bit Guest-state fields. */
4320	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4321	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4322	case VMX_VMCS64_GUEST_PAT_FULL:
4323	case VMX_VMCS64_GUEST_EFER_FULL:
4324	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4325	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4326	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4327	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4328	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4329	/* 64-bit Host-state fields. */
4330	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4331	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4332	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4333	{
4334	rc = VMXWriteVmcs32(idxField, u64Val);
4335	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4336	break;
4337	}
4338
4339	/*
4340	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4341	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4342	*/
4343	/* Natural-width Guest-state fields. */
4344	case VMX_VMCS_GUEST_CR3:
4345	case VMX_VMCS_GUEST_ES_BASE:
4346	case VMX_VMCS_GUEST_CS_BASE:
4347	case VMX_VMCS_GUEST_SS_BASE:
4348	case VMX_VMCS_GUEST_DS_BASE:
4349	case VMX_VMCS_GUEST_FS_BASE:
4350	case VMX_VMCS_GUEST_GS_BASE:
4351	case VMX_VMCS_GUEST_LDTR_BASE:
4352	case VMX_VMCS_GUEST_TR_BASE:
4353	case VMX_VMCS_GUEST_GDTR_BASE:
4354	case VMX_VMCS_GUEST_IDTR_BASE:
4355	case VMX_VMCS_GUEST_DR7:
4356	case VMX_VMCS_GUEST_RSP:
4357	case VMX_VMCS_GUEST_RIP:
4358	case VMX_VMCS_GUEST_SYSENTER_ESP:
4359	case VMX_VMCS_GUEST_SYSENTER_EIP:
4360	{
4361	if (!(u64Val >> 32))
4362	{
4363	/* If this field is 64-bit, VT-x will zero out the top bits. */
4364	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4365	}
4366	else
4367	{
4368	/* Assert that only the 32->64 switcher case should ever come here. */
4369	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4370	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4371	}
4372	break;
4373	}
4374
4375	default:
4376	{
4377	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4378	rc = VERR_INVALID_PARAMETER;
4379	break;
4380	}
4381	}
4382	AssertRCReturn(rc, rc);
4383	return rc;
4384	}
4385
4386
4387	/**
4388	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4389	* hosts (except darwin) for 64-bit guests.
4390	*
4391	* @param pVCpu Pointer to the VMCPU.
4392	* @param idxField The VMCS field encoding.
4393	* @param u64Val 16, 32 or 64 bits value.
4394	*/
4395	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4396	{
4397	AssertPtr(pVCpu);
4398	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4399
4400	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4401	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4402
4403	/* Make sure there are no duplicates. */
4404	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4405	{
4406	if (pCache->Write.aField[i] == idxField)
4407	{
4408	pCache->Write.aFieldVal[i] = u64Val;
4409	return VINF_SUCCESS;
4410	}
4411	}
4412
4413	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4414	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4415	pCache->Write.cValidEntries++;
4416	return VINF_SUCCESS;
4417	}
4418
4419	/* Enable later when the assembly code uses these as callbacks. */
4420	#if 0
4421	/*
4422	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4423	*
4424	* @param pVCpu Pointer to the VMCPU.
4425	* @param pCache Pointer to the VMCS cache.
4426	*
4427	* @remarks No-long-jump zone!!!
4428	*/
4429	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4430	{
4431	AssertPtr(pCache);
4432	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4433	{
4434	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4435	AssertRC(rc);
4436	}
4437	pCache->Write.cValidEntries = 0;
4438	}
4439
4440
4441	/**
4442	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4443	*
4444	* @param pVCpu Pointer to the VMCPU.
4445	* @param pCache Pointer to the VMCS cache.
4446	*
4447	* @remarks No-long-jump zone!!!
4448	*/
4449	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4450	{
4451	AssertPtr(pCache);
4452	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4453	{
4454	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4455	AssertRC(rc);
4456	}
4457	}
4458	#endif
4459	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4460
4461
4462	/**
4463	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4464	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4465	* timer.
4466	*
4467	* @returns VBox status code.
4468	* @param pVCpu Pointer to the VMCPU.
4469	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4470	* out-of-sync. Make sure to update the required fields
4471	* before using them.
4472	* @remarks No-long-jump zone!!!
4473	*/
4474	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4475	{
4476	int rc = VERR_INTERNAL_ERROR_5;
4477	bool fOffsettedTsc = false;
4478	PVM pVM = pVCpu->CTX_SUFF(pVM);
4479	if (pVM->hm.s.vmx.fUsePreemptTimer)
4480	{
4481	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4482
4483	/* Make sure the returned values have sane upper and lower boundaries. */
4484	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4485	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4486	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4487	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4488
4489	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4490	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4491	}
4492	else
4493	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4494
4495	if (fOffsettedTsc)
4496	{
4497	uint64_t u64CurTSC = ASMReadTSC();
4498	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4499	{
4500	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4501	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4502
4503	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4504	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4505	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4506	}
4507	else
4508	{
4509	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4510	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4511	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4512	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4513	}
4514	}
4515	else
4516	{
4517	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4518	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4519	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4520	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4521	}
4522	}
4523
4524
4525	/**
4526	* Determines if an exception is a contributory exception. Contributory
4527	* exceptions are ones which can cause double-faults. Page-fault is
4528	* intentionally not included here as it's a conditional contributory exception.
4529	*
4530	* @returns true if the exception is contributory, false otherwise.
4531	* @param uVector The exception vector.
4532	*/
4533	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4534	{
4535	switch (uVector)
4536	{
4537	case X86_XCPT_GP:
4538	case X86_XCPT_SS:
4539	case X86_XCPT_NP:
4540	case X86_XCPT_TS:
4541	case X86_XCPT_DE:
4542	return true;
4543	default:
4544	break;
4545	}
4546	return false;
4547	}
4548
4549
4550	/**
4551	* Sets an event as a pending event to be injected into the guest.
4552	*
4553	* @param pVCpu Pointer to the VMCPU.
4554	* @param u32IntrInfo The VM-entry interruption-information field.
4555	* @param cbInstr The VM-entry instruction length in bytes (for software
4556	* interrupts, exceptions and privileged software
4557	* exceptions).
4558	* @param u32ErrCode The VM-entry exception error code.
4559	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4560	* page-fault.
4561	*/
4562	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4563	RTGCUINTPTR GCPtrFaultAddress)
4564	{
4565	Assert(!pVCpu->hm.s.Event.fPending);
4566	pVCpu->hm.s.Event.fPending = true;
4567	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4568	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4569	pVCpu->hm.s.Event.cbInstr = cbInstr;
4570	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4571	}
4572
4573
4574	/**
4575	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4576	*
4577	* @param pVCpu Pointer to the VMCPU.
4578	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4579	* out-of-sync. Make sure to update the required fields
4580	* before using them.
4581	*/
4582	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4583	{
4584	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4585	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4586	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4587	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4588	}
4589
4590
4591	/**
4592	* Handle a condition that occurred while delivering an event through the guest
4593	* IDT.
4594	*
4595	* @returns VBox status code (informational error codes included).
4596	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4597	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4598	* continue execution of the guest which will delivery the #DF.
4599	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4600	*
4601	* @param pVCpu Pointer to the VMCPU.
4602	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4603	* out-of-sync. Make sure to update the required fields
4604	* before using them.
4605	* @param pVmxTransient Pointer to the VMX transient structure.
4606	*
4607	* @remarks No-long-jump zone!!!
4608	*/
4609	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4610	{
4611	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4612	AssertRC(rc);
4613	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4614	{
4615	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4616	AssertRCReturn(rc, rc);
4617
4618	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4619	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4620	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4621
4622	typedef enum
4623	{
4624	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4625	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4626	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4627	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4628	} VMXREFLECTXCPT;
4629
4630	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4631	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4632	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4633	{
4634	enmReflect = VMXREFLECTXCPT_XCPT;
4635	#ifdef VBOX_STRICT
4636	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4637	&& uExitVector == X86_XCPT_PF)
4638	{
4639	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4640	}
4641	#endif
4642	if ( uExitVector == X86_XCPT_PF
4643	&& uIdtVector == X86_XCPT_PF)
4644	{
4645	pVmxTransient->fVectoringPF = true;
4646	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4647	}
4648	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4649	&& hmR0VmxIsContributoryXcpt(uExitVector)
4650	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4651	\|\| uIdtVector == X86_XCPT_PF))
4652	{
4653	enmReflect = VMXREFLECTXCPT_DF;
4654	}
4655	else if (uIdtVector == X86_XCPT_DF)
4656	enmReflect = VMXREFLECTXCPT_TF;
4657	}
4658	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4659	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4660	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4661	{
4662	/*
4663	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4664	* (whatever they are) as they reoccur when restarting the instruction.
4665	*/
4666	enmReflect = VMXREFLECTXCPT_XCPT;
4667	}
4668
4669	switch (enmReflect)
4670	{
4671	case VMXREFLECTXCPT_XCPT:
4672	{
4673	uint32_t u32ErrCode = 0;
4674	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4675	{
4676	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4677	AssertRCReturn(rc, rc);
4678	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4679	}
4680
4681	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4682	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4683	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4684	rc = VINF_SUCCESS;
4685	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4686	pVCpu->hm.s.Event.u32ErrCode));
4687	break;
4688	}
4689
4690	case VMXREFLECTXCPT_DF:
4691	{
4692	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4693	rc = VINF_VMX_DOUBLE_FAULT;
4694	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4695	uIdtVector, uExitVector));
4696	break;
4697	}
4698
4699	case VMXREFLECTXCPT_TF:
4700	{
4701	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4702	rc = VINF_EM_RESET;
4703	break;
4704	}
4705
4706	default:
4707	Assert(rc == VINF_SUCCESS);
4708	break;
4709	}
4710	}
4711	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4712	return rc;
4713	}
4714
4715
4716	/**
4717	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4718	*
4719	* @returns VBox status code.
4720	* @param pVCpu Pointer to the VMCPU.
4721	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4722	* out-of-sync. Make sure to update the required fields
4723	* before using them.
4724	*
4725	* @remarks No-long-jump zone!!!
4726	*/
4727	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4728	{
4729	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4730	{
4731	uint32_t uVal = 0;
4732	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4733	AssertRCReturn(rc, rc);
4734	uint32_t uShadow = 0;
4735	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4736	AssertRCReturn(rc, rc);
4737
4738	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4739	CPUMSetGuestCR0(pVCpu, uVal);
4740	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4741	}
4742	return VINF_SUCCESS;
4743	}
4744
4745
4746	/**
4747	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4748	*
4749	* @returns VBox status code.
4750	* @param pVCpu Pointer to the VMCPU.
4751	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4752	* out-of-sync. Make sure to update the required fields
4753	* before using them.
4754	*
4755	* @remarks No-long-jump zone!!!
4756	*/
4757	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4758	{
4759	int rc = VINF_SUCCESS;
4760	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4761	{
4762	uint32_t uVal = 0;
4763	uint32_t uShadow = 0;
4764	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4765	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4766	AssertRCReturn(rc, rc);
4767	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4768	CPUMSetGuestCR4(pVCpu, uVal);
4769	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4770	}
4771	return rc;
4772	}
4773
4774
4775	/**
4776	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4777	*
4778	* @returns VBox status code.
4779	* @param pVCpu Pointer to the VMCPU.
4780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4781	* out-of-sync. Make sure to update the required fields
4782	* before using them.
4783	*
4784	* @remarks No-long-jump zone!!!
4785	*/
4786	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4787	{
4788	int rc = VINF_SUCCESS;
4789	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4790	{
4791	RTGCUINTREG uVal = 0;
4792	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4793	AssertRCReturn(rc, rc);
4794	pMixedCtx->rip = uVal;
4795	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4796	}
4797	return rc;
4798	}
4799
4800
4801	/**
4802	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4803	*
4804	* @returns VBox status code.
4805	* @param pVCpu Pointer to the VMCPU.
4806	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4807	* out-of-sync. Make sure to update the required fields
4808	* before using them.
4809	*
4810	* @remarks No-long-jump zone!!!
4811	*/
4812	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4813	{
4814	int rc = VINF_SUCCESS;
4815	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4816	{
4817	RTGCUINTREG uVal = 0;
4818	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4819	AssertRCReturn(rc, rc);
4820	pMixedCtx->rsp = uVal;
4821	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4822	}
4823	return rc;
4824	}
4825
4826
4827	/**
4828	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4829	*
4830	* @returns VBox status code.
4831	* @param pVCpu Pointer to the VMCPU.
4832	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4833	* out-of-sync. Make sure to update the required fields
4834	* before using them.
4835	*
4836	* @remarks No-long-jump zone!!!
4837	*/
4838	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4839	{
4840	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4841	{
4842	uint32_t uVal = 0;
4843	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4844	AssertRCReturn(rc, rc);
4845	pMixedCtx->eflags.u32 = uVal;
4846
4847	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4848	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4849	{
4850	PVM pVM = pVCpu->CTX_SUFF(pVM);
4851	Assert(pVM->hm.s.vmx.pRealModeTSS);
4852	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4853	pMixedCtx->eflags.Bits.u1VM = 0;
4854	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4855	}
4856
4857	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4858	}
4859	return VINF_SUCCESS;
4860	}
4861
4862
4863	/**
4864	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4865	* guest-CPU context.
4866	*/
4867	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4868	{
4869	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4870	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4871	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4872	return rc;
4873	}
4874
4875
4876	/**
4877	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4878	* from the guest-state area in the VMCS.
4879	*
4880	* @param pVCpu Pointer to the VMCPU.
4881	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4882	* out-of-sync. Make sure to update the required fields
4883	* before using them.
4884	*
4885	* @remarks No-long-jump zone!!!
4886	*/
4887	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4888	{
4889	uint32_t uIntrState = 0;
4890	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4891	AssertRC(rc);
4892
4893	if (!uIntrState)
4894	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4895	else
4896	{
4897	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4898	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4899	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4900	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4901	AssertRC(rc);
4902	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4903	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4904	}
4905	}
4906
4907
4908	/**
4909	* Saves the guest's activity state.
4910	*
4911	* @returns VBox status code.
4912	* @param pVCpu Pointer to the VMCPU.
4913	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4914	* out-of-sync. Make sure to update the required fields
4915	* before using them.
4916	*
4917	* @remarks No-long-jump zone!!!
4918	*/
4919	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4920	{
4921	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4922	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4923	return VINF_SUCCESS;
4924	}
4925
4926
4927	/**
4928	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4929	* the current VMCS into the guest-CPU context.
4930	*
4931	* @returns VBox status code.
4932	* @param pVCpu Pointer to the VMCPU.
4933	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4934	* out-of-sync. Make sure to update the required fields
4935	* before using them.
4936	*
4937	* @remarks No-long-jump zone!!!
4938	*/
4939	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4940	{
4941	int rc = VINF_SUCCESS;
4942	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4943	{
4944	uint32_t u32Val = 0;
4945	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4946	pMixedCtx->SysEnter.cs = u32Val;
4947	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4948	}
4949
4950	RTGCUINTREG uGCVal = 0;
4951	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4952	{
4953	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4954	pMixedCtx->SysEnter.eip = uGCVal;
4955	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4956	}
4957	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4958	{
4959	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4960	pMixedCtx->SysEnter.esp = uGCVal;
4961	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4962	}
4963	return rc;
4964	}
4965
4966
4967	/**
4968	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4969	* context.
4970	*
4971	* @returns VBox status code.
4972	* @param pVCpu Pointer to the VMCPU.
4973	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4974	* out-of-sync. Make sure to update the required fields
4975	* before using them.
4976	*
4977	* @remarks No-long-jump zone!!!
4978	*/
4979	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4980	{
4981	int rc = VINF_SUCCESS;
4982	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4983	{
4984	RTGCUINTREG uVal = 0;
4985	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4986	pMixedCtx->fs.u64Base = uVal;
4987	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4988	}
4989	return rc;
4990	}
4991
4992
4993	/**
4994	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4995	* context.
4996	*
4997	* @returns VBox status code.
4998	* @param pVCpu Pointer to the VMCPU.
4999	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5000	* out-of-sync. Make sure to update the required fields
5001	* before using them.
5002	*
5003	* @remarks No-long-jump zone!!!
5004	*/
5005	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5006	{
5007	int rc = VINF_SUCCESS;
5008	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5009	{
5010	RTGCUINTREG uVal = 0;
5011	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5012	pMixedCtx->gs.u64Base = uVal;
5013	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5014	}
5015	return rc;
5016	}
5017
5018
5019	/**
5020	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5021	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5022	*
5023	* @returns VBox status code.
5024	* @param pVCpu Pointer to the VMCPU.
5025	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5026	* out-of-sync. Make sure to update the required fields
5027	* before using them.
5028	*
5029	* @remarks No-long-jump zone!!!
5030	*/
5031	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5032	{
5033	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5034	return VINF_SUCCESS;
5035
5036	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5037	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5038	{
5039	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5040	pMsr += i;
5041	switch (pMsr->u32IndexMSR)
5042	{
5043	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5044	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5045	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5046	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5047	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5048	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5049	default:
5050	{
5051	AssertFailed();
5052	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5053	}
5054	}
5055	}
5056	#endif
5057
5058	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5059	return VINF_SUCCESS;
5060	}
5061
5062
5063	/**
5064	* Saves the guest control registers from the current VMCS into the guest-CPU
5065	* context.
5066	*
5067	* @returns VBox status code.
5068	* @param pVCpu Pointer to the VMCPU.
5069	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5070	* out-of-sync. Make sure to update the required fields
5071	* before using them.
5072	*
5073	* @remarks No-long-jump zone!!!
5074	*/
5075	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5076	{
5077	/* Guest CR0. Guest FPU. */
5078	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5079
5080	/* Guest CR4. */
5081	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5082	AssertRCReturn(rc, rc);
5083
5084	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5085	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5086	{
5087	PVM pVM = pVCpu->CTX_SUFF(pVM);
5088	if ( pVM->hm.s.fNestedPaging
5089	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5090	{
5091	RTGCUINTREG uVal = 0;
5092	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5093	if (pMixedCtx->cr3 != uVal)
5094	{
5095	CPUMSetGuestCR3(pVCpu, uVal);
5096	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5097	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5098	}
5099
5100	/* We require EFER to check PAE mode. */
5101	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5102
5103	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5104	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5105	{
5106	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5107	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5108	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5109	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5110	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5111	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5112	}
5113	AssertRCReturn(rc, rc);
5114	}
5115
5116	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5117	}
5118	return rc;
5119	}
5120
5121
5122	/**
5123	* Reads a guest segment register from the current VMCS into the guest-CPU
5124	* context.
5125	*
5126	* @returns VBox status code.
5127	* @param pVCpu Pointer to the VMCPU.
5128	* @param idxSel Index of the selector in the VMCS.
5129	* @param idxLimit Index of the segment limit in the VMCS.
5130	* @param idxBase Index of the segment base in the VMCS.
5131	* @param idxAccess Index of the access rights of the segment in the VMCS.
5132	* @param pSelReg Pointer to the segment selector.
5133	*
5134	* @remarks No-long-jump zone!!!
5135	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5136	* as that takes care of whether to read from the VMCS cache or not.
5137	*/
5138	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5139	PCPUMSELREG pSelReg)
5140	{
5141	uint32_t u32Val = 0;
5142	int rc = VMXReadVmcs32(idxSel, &u32Val);
5143	AssertRCReturn(rc, rc);
5144	pSelReg->Sel = (uint16_t)u32Val;
5145	pSelReg->ValidSel = (uint16_t)u32Val;
5146	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5147
5148	rc = VMXReadVmcs32(idxLimit, &u32Val);
5149	AssertRCReturn(rc, rc);
5150	pSelReg->u32Limit = u32Val;
5151
5152	RTGCUINTREG uGCVal = 0;
5153	rc = VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5154	AssertRCReturn(rc, rc);
5155	pSelReg->u64Base = uGCVal;
5156
5157	rc = VMXReadVmcs32(idxAccess, &u32Val);
5158	AssertRCReturn(rc, rc);
5159	pSelReg->Attr.u = u32Val;
5160
5161	/*
5162	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5163	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5164	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5165	*
5166	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5167	*/
5168	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5169	{
5170	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5171	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5172	}
5173	return VINF_SUCCESS;
5174	}
5175
5176	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5177	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5178	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5179	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5180	#else
5181	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5182	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5183	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5184	#endif
5185
5186	/**
5187	* Saves the guest segment registers from the current VMCS into the guest-CPU
5188	* context.
5189	*
5190	* @returns VBox status code.
5191	* @param pVCpu Pointer to the VMCPU.
5192	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5193	* out-of-sync. Make sure to update the required fields
5194	* before using them.
5195	*
5196	* @remarks No-long-jump zone!!!
5197	*/
5198	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5199	{
5200	/* Guest segment registers. */
5201	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5202	{
5203	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5204	AssertRCReturn(rc, rc);
5205	rc = VMXLOCAL_READ_SEG(CS, cs);
5206	AssertRCReturn(rc, rc);
5207	rc = VMXLOCAL_READ_SEG(SS, ss);
5208	AssertRCReturn(rc, rc);
5209	rc = VMXLOCAL_READ_SEG(DS, ds);
5210	AssertRCReturn(rc, rc);
5211	rc = VMXLOCAL_READ_SEG(ES, es);
5212	AssertRCReturn(rc, rc);
5213	rc = VMXLOCAL_READ_SEG(FS, fs);
5214	AssertRCReturn(rc, rc);
5215	rc = VMXLOCAL_READ_SEG(GS, gs);
5216	AssertRCReturn(rc, rc);
5217
5218	/* Restore segment attributes for real-on-v86 mode hack. */
5219	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5220	{
5221	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5222	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5223	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5224	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5225	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5226	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5227	}
5228	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5229	}
5230
5231	return VINF_SUCCESS;
5232	}
5233
5234
5235	/**
5236	* Saves the guest descriptor table registers and task register from the current
5237	* VMCS into the guest-CPU context.
5238	*
5239	* @returns VBox status code.
5240	* @param pVCpu Pointer to the VMCPU.
5241	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5242	* out-of-sync. Make sure to update the required fields
5243	* before using them.
5244	*
5245	* @remarks No-long-jump zone!!!
5246	*/
5247	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5248	{
5249	int rc = VINF_SUCCESS;
5250
5251	/* Guest LDTR. */
5252	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5253	{
5254	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5255	AssertRCReturn(rc, rc);
5256	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5257	}
5258
5259	/* Guest GDTR. */
5260	RTGCUINTREG uGCVal = 0;
5261	uint32_t u32Val = 0;
5262	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5263	{
5264	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5265	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5266	pMixedCtx->gdtr.pGdt = uGCVal;
5267	pMixedCtx->gdtr.cbGdt = u32Val;
5268	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5269	}
5270
5271	/* Guest IDTR. */
5272	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5273	{
5274	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5275	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5276	pMixedCtx->idtr.pIdt = uGCVal;
5277	pMixedCtx->idtr.cbIdt = u32Val;
5278	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5279	}
5280
5281	/* Guest TR. */
5282	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5283	{
5284	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5285
5286	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5287	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5288	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5289	AssertRCReturn(rc, rc);
5290	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5291	}
5292	return rc;
5293	}
5294
5295	#undef VMXLOCAL_READ_SEG
5296
5297
5298	/**
5299	* Saves the guest debug registers from the current VMCS into the guest-CPU
5300	* context.
5301	*
5302	* @returns VBox status code.
5303	* @param pVCpu Pointer to the VMCPU.
5304	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5305	* out-of-sync. Make sure to update the required fields
5306	* before using them.
5307	*
5308	* @remarks No-long-jump zone!!!
5309	*/
5310	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5311	{
5312	int rc = VINF_SUCCESS;
5313	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5314	{
5315	RTGCUINTREG uVal;
5316	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5317	pMixedCtx->dr[7] = uVal;
5318
5319	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5320	}
5321	return rc;
5322	}
5323
5324
5325	/**
5326	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5327	*
5328	* @returns VBox status code.
5329	* @param pVCpu Pointer to the VMCPU.
5330	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331	* out-of-sync. Make sure to update the required fields
5332	* before using them.
5333	*
5334	* @remarks No-long-jump zone!!!
5335	*/
5336	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337	{
5338	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5339	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5340	return VINF_SUCCESS;
5341	}
5342
5343
5344	/**
5345	* Saves the entire guest state from the currently active VMCS into the
5346	* guest-CPU context. This essentially VMREADs all guest-data.
5347	*
5348	* @returns VBox status code.
5349	* @param pVCpu Pointer to the VMCPU.
5350	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5351	* out-of-sync. Make sure to update the required fields
5352	* before using them.
5353	*/
5354	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5355	{
5356	Assert(pVCpu);
5357	Assert(pMixedCtx);
5358
5359	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5360	return VINF_SUCCESS;
5361
5362	VMMRZCallRing3Disable(pVCpu);
5363	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5364	LogFunc(("\n"));
5365
5366	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5367	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5368
5369	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5370	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5371
5372	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5373	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5374
5375	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5376	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5377
5378	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5379	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5380
5381	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5383
5384	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5385	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5386
5387	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5388	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5389
5390	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5391	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5392
5393	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5394	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5395
5396	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5397	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5398
5399	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5400	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5401
5402	VMMRZCallRing3Enable(pVCpu);
5403	return rc;
5404	}
5405
5406
5407	/**
5408	* Check per-VM and per-VCPU force flag actions that require us to go back to
5409	* ring-3 for one reason or another.
5410	*
5411	* @returns VBox status code (information status code included).
5412	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5413	* ring-3.
5414	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5415	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5416	* interrupts)
5417	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5418	* all EMTs to be in ring-3.
5419	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5420	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5421	* to the EM loop.
5422	*
5423	* @param pVM Pointer to the VM.
5424	* @param pVCpu Pointer to the VMCPU.
5425	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5426	* out-of-sync. Make sure to update the required fields
5427	* before using them.
5428	*/
5429	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5430	{
5431	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5432
5433	int rc = VERR_INTERNAL_ERROR_5;
5434	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5435	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5436	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5437	{
5438	/* We need the control registers now, make sure the guest-CPU context is updated. */
5439	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5440	AssertRCReturn(rc, rc);
5441
5442	/* Pending HM CR3 sync. */
5443	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5444	{
5445	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5446	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5447	}
5448	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5449	{
5450	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5451	AssertRC(rc);
5452	}
5453
5454	/* Pending PGM C3 sync. */
5455	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5456	{
5457	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5458	if (rc != VINF_SUCCESS)
5459	{
5460	AssertRC(rc);
5461	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5462	return rc;
5463	}
5464	}
5465
5466	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5467	/* -XXX- what was that about single stepping? */
5468	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5469	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5470	{
5471	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5472	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5473	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5474	return rc;
5475	}
5476
5477	/* Pending VM request packets, such as hardware interrupts. */
5478	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5479	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5480	{
5481	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5482	return VINF_EM_PENDING_REQUEST;
5483	}
5484
5485	/* Pending PGM pool flushes. */
5486	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5487	{
5488	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5489	return VINF_PGM_POOL_FLUSH_PENDING;
5490	}
5491
5492	/* Pending DMA requests. */
5493	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5494	{
5495	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5496	return VINF_EM_RAW_TO_R3;
5497	}
5498	}
5499
5500	/* Paranoia. */
5501	Assert(rc != VERR_EM_INTERPRETER);
5502	return VINF_SUCCESS;
5503	}
5504
5505
5506	/**
5507	* Converts any TRPM trap into a pending VMX event. This is typically used when
5508	* entering from ring-3 (not longjmp returns).
5509	*
5510	* @param pVCpu Pointer to the VMCPU.
5511	*/
5512	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5513	{
5514	Assert(TRPMHasTrap(pVCpu));
5515	Assert(!pVCpu->hm.s.Event.fPending);
5516
5517	uint8_t uVector;
5518	TRPMEVENT enmTrpmEvent;
5519	RTGCUINT uErrCode;
5520	RTGCUINTPTR GCPtrFaultAddress;
5521	uint8_t cbInstr;
5522
5523	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5524	AssertRC(rc);
5525
5526	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5527	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5528	if (enmTrpmEvent == TRPM_TRAP)
5529	{
5530	switch (uVector)
5531	{
5532	case X86_XCPT_BP:
5533	case X86_XCPT_OF:
5534	{
5535	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5536	break;
5537	}
5538
5539	case X86_XCPT_PF:
5540	case X86_XCPT_DF:
5541	case X86_XCPT_TS:
5542	case X86_XCPT_NP:
5543	case X86_XCPT_SS:
5544	case X86_XCPT_GP:
5545	case X86_XCPT_AC:
5546	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5547	/* no break! */
5548	default:
5549	{
5550	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5551	break;
5552	}
5553	}
5554	}
5555	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5556	{
5557	if (uVector != X86_XCPT_NMI)
5558	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5559	else
5560	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5561	}
5562	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5563	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5564	else
5565	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5566
5567	rc = TRPMResetTrap(pVCpu);
5568	AssertRC(rc);
5569	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5570	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5571	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5572	}
5573
5574
5575	/**
5576	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5577	* VT-x to execute any instruction.
5578	*
5579	* @param pvCpu Pointer to the VMCPU.
5580	*/
5581	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5582	{
5583	Assert(pVCpu->hm.s.Event.fPending);
5584
5585	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5586	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5587	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5588	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5589
5590	/* If a trap was already pending, we did something wrong! */
5591	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5592
5593	TRPMEVENT enmTrapType;
5594	switch (uVectorType)
5595	{
5596	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5597	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5598	enmTrapType = TRPM_HARDWARE_INT;
5599	break;
5600	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5601	enmTrapType = TRPM_SOFTWARE_INT;
5602	break;
5603	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5604	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5605	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5606	enmTrapType = TRPM_TRAP;
5607	break;
5608	default:
5609	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5610	enmTrapType = TRPM_32BIT_HACK;
5611	break;
5612	}
5613
5614	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5615	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5616	AssertRC(rc);
5617
5618	if (fErrorCodeValid)
5619	TRPMSetErrorCode(pVCpu, uErrorCode);
5620	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5621	&& uVector == X86_XCPT_PF)
5622	{
5623	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5624	}
5625	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5626	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5627	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5628	{
5629	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5630	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5631	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5632	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5633	}
5634	pVCpu->hm.s.Event.fPending = false;
5635	}
5636
5637
5638	/**
5639	* Does the necessary state syncing before doing a longjmp to ring-3.
5640	*
5641	* @param pVM Pointer to the VM.
5642	* @param pVCpu Pointer to the VMCPU.
5643	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5644	* out-of-sync. Make sure to update the required fields
5645	* before using them.
5646	* @param rcExit The reason for exiting to ring-3. Can be
5647	* VINF_VMM_UNKNOWN_RING3_CALL.
5648	*
5649	* @remarks No-long-jmp zone!!!
5650	*/
5651	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5652	{
5653	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5654	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5655
5656	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5657	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5658	AssertRC(rc);
5659
5660	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5661	if (CPUMIsGuestFPUStateActive(pVCpu))
5662	{
5663	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5664	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5665	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5666	}
5667
5668	/* Restore debug registers if necessary and resync on next R0 reentry. */
5669	if (CPUMIsGuestDebugStateActive(pVCpu))
5670	{
5671	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5672	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5673	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5674	}
5675	else if (CPUMIsHyperDebugStateActive(pVCpu))
5676	{
5677	CPUMR0LoadHostDebugState(pVM, pVCpu);
5678	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5679	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5680	}
5681
5682	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5683	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5684	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5685	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5686	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5687	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5688	}
5689
5690
5691	/**
5692	* An action requires us to go back to ring-3. This function does the necessary
5693	* steps before we can safely return to ring-3. This is not the same as longjmps
5694	* to ring-3, this is voluntary.
5695	*
5696	* @param pVM Pointer to the VM.
5697	* @param pVCpu Pointer to the VMCPU.
5698	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5699	* out-of-sync. Make sure to update the required fields
5700	* before using them.
5701	* @param rcExit The reason for exiting to ring-3. Can be
5702	* VINF_VMM_UNKNOWN_RING3_CALL.
5703	*/
5704	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5705	{
5706	Assert(pVM);
5707	Assert(pVCpu);
5708	Assert(pMixedCtx);
5709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5710
5711	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5712	{
5713	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5714	return;
5715	}
5716	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5717	{
5718	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5719	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5720	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5721	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5722	return;
5723	}
5724
5725	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5726	VMMRZCallRing3Disable(pVCpu);
5727	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5728
5729	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5730	if (pVCpu->hm.s.Event.fPending)
5731	{
5732	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5733	Assert(!pVCpu->hm.s.Event.fPending);
5734	}
5735
5736	/* Sync. the guest state. */
5737	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5738	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5739
5740	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5741	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5742	\| CPUM_CHANGED_LDTR
5743	\| CPUM_CHANGED_GDTR
5744	\| CPUM_CHANGED_IDTR
5745	\| CPUM_CHANGED_TR
5746	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5747
5748	/* On our way back from ring-3 the following needs to be done. */
5749	/** @todo This can change with preemption hooks. */
5750	if (rcExit == VINF_EM_RAW_INTERRUPT)
5751	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5752	else
5753	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5754
5755	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5756	VMMRZCallRing3Enable(pVCpu);
5757	}
5758
5759
5760	/**
5761	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5762	* longjump to ring-3 and possibly get preempted.
5763	*
5764	* @param pVCpu Pointer to the VMCPU.
5765	* @param enmOperation The operation causing the ring-3 longjump.
5766	* @param pvUser The user argument (pointer to the possibly
5767	* out-of-date guest-CPU context).
5768	*
5769	* @remarks Must never be called with @a enmOperation ==
5770	* VMMCALLRING3_VM_R0_ASSERTION.
5771	*/
5772	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5773	{
5774	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5775	Assert(pVCpu);
5776	Assert(pvUser);
5777	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5778	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5779
5780	VMMRZCallRing3Disable(pVCpu);
5781	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5782	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5783	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5784	VMMRZCallRing3Enable(pVCpu);
5785	}
5786
5787
5788	/**
5789	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5790	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5791	*
5792	* @returns VBox status code.
5793	* @param pVCpu Pointer to the VMCPU.
5794	*/
5795	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5796	{
5797	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5798	{
5799	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5800	{
5801	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5802	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5803	AssertRC(rc);
5804	}
5805	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5806	}
5807
5808
5809	/**
5810	* Injects any pending events into the guest if the guest is in a state to
5811	* receive them.
5812	*
5813	* @returns VBox status code (informational status codes included).
5814	* @param pVCpu Pointer to the VMCPU.
5815	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5816	* out-of-sync. Make sure to update the required fields
5817	* before using them.
5818	*/
5819	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5820	{
5821	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5822	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5823	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5824	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5825
5826	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5827	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5828	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5829	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5830	Assert(!TRPMHasTrap(pVCpu));
5831
5832	int rc = VINF_SUCCESS;
5833	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5834	{
5835	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5836	bool fInject = true;
5837	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5838	{
5839	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5840	AssertRCReturn(rc, rc);
5841	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5842	if ( fBlockInt
5843	\|\| fBlockSti
5844	\|\| fBlockMovSS)
5845	{
5846	fInject = false;
5847	}
5848	}
5849	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5850	&& ( fBlockMovSS
5851	\|\| fBlockSti))
5852	{
5853	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5854	fInject = false;
5855	}
5856
5857	if (fInject)
5858	{
5859	Log(("Injecting pending event\n"));
5860	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5861	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5862	AssertRCReturn(rc, rc);
5863	pVCpu->hm.s.Event.fPending = false;
5864	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5865	}
5866	else
5867	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5868	} /** @todo SMI. SMIs take priority over NMIs. */
5869	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5870	{
5871	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5872	if ( !fBlockMovSS
5873	&& !fBlockSti)
5874	{
5875	Log(("Injecting NMI\n"));
5876	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5877	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5878	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5879	0 /* GCPtrFaultAddress */, &uIntrState);
5880	AssertRCReturn(rc, rc);
5881	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5882	}
5883	else
5884	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5885	}
5886	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5887	{
5888	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5889	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5890	AssertRCReturn(rc, rc);
5891	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5892	if ( !fBlockInt
5893	&& !fBlockSti
5894	&& !fBlockMovSS)
5895	{
5896	uint8_t u8Interrupt;
5897	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5898	if (RT_SUCCESS(rc))
5899	{
5900	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5901	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5902	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5903	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5904	0 /* GCPtrFaultAddress */, &uIntrState);
5905	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5906	}
5907	else
5908	{
5909	/** @todo Does this actually happen? If not turn it into an assertion. */
5910	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5911	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5912	rc = VINF_SUCCESS;
5913	}
5914	}
5915	else
5916	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5917	}
5918
5919	/*
5920	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5921	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5922	*/
5923	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5924	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5925	int rc2 = VINF_SUCCESS;
5926	if ( fBlockSti
5927	\|\| fBlockMovSS)
5928	{
5929	if (!DBGFIsStepping(pVCpu))
5930	{
5931	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5932	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5933	{
5934	/*
5935	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5936	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5937	*/
5938	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5939	}
5940	}
5941	else
5942	{
5943	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5944	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5945	uIntrState = 0;
5946	}
5947	}
5948
5949	/*
5950	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5951	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5952	*/
5953	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5954	AssertRC(rc2);
5955
5956	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5957	return rc;
5958	}
5959
5960
5961	/**
5962	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5963	*
5964	* @param pVCpu Pointer to the VMCPU.
5965	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5966	* out-of-sync. Make sure to update the required fields
5967	* before using them.
5968	*/
5969	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5970	{
5971	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5972	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5973	}
5974
5975
5976	/**
5977	* Injects a double-fault (#DF) exception into the VM.
5978	*
5979	* @returns VBox status code (informational status code included).
5980	* @param pVCpu Pointer to the VMCPU.
5981	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5982	* out-of-sync. Make sure to update the required fields
5983	* before using them.
5984	*/
5985	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5986	{
5987	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5988	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5989	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5990	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5991	puIntrState);
5992	}
5993
5994
5995	/**
5996	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5997	*
5998	* @param pVCpu Pointer to the VMCPU.
5999	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6000	* out-of-sync. Make sure to update the required fields
6001	* before using them.
6002	*/
6003	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6004	{
6005	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6006	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6007	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6008	}
6009
6010
6011	/**
6012	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6013	*
6014	* @param pVCpu Pointer to the VMCPU.
6015	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6016	* out-of-sync. Make sure to update the required fields
6017	* before using them.
6018	* @param cbInstr The value of RIP that is to be pushed on the guest
6019	* stack.
6020	*/
6021	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6022	{
6023	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6024	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6025	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6026	}
6027
6028
6029	/**
6030	* Injects a general-protection (#GP) fault into the VM.
6031	*
6032	* @returns VBox status code (informational status code included).
6033	* @param pVCpu Pointer to the VMCPU.
6034	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6035	* out-of-sync. Make sure to update the required fields
6036	* before using them.
6037	* @param u32ErrorCode The error code associated with the #GP.
6038	*/
6039	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6040	uint32_t *puIntrState)
6041	{
6042	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6043	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6044	if (fErrorCodeValid)
6045	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6046	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6047	puIntrState);
6048	}
6049
6050
6051	/**
6052	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6053	*
6054	* @param pVCpu Pointer to the VMCPU.
6055	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6056	* out-of-sync. Make sure to update the required fields
6057	* before using them.
6058	* @param uVector The software interrupt vector number.
6059	* @param cbInstr The value of RIP that is to be pushed on the guest
6060	* stack.
6061	*/
6062	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6063	{
6064	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6065	if ( uVector == X86_XCPT_BP
6066	\|\| uVector == X86_XCPT_OF)
6067	{
6068	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6069	}
6070	else
6071	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6072	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6073	}
6074
6075
6076	/**
6077	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6078	* stack.
6079	*
6080	* @returns VBox status code (information status code included).
6081	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6082	* @param pVM Pointer to the VM.
6083	* @param pMixedCtx Pointer to the guest-CPU context.
6084	* @param uValue The value to push to the guest stack.
6085	*/
6086	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6087	{
6088	/*
6089	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6090	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6091	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6092	*/
6093	if (pMixedCtx->sp == 1)
6094	return VINF_EM_RESET;
6095	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6096	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6097	AssertRCReturn(rc, rc);
6098	return rc;
6099	}
6100
6101
6102	/**
6103	* Injects an event into the guest upon VM-entry by updating the relevant fields
6104	* in the VM-entry area in the VMCS.
6105	*
6106	* @returns VBox status code (informational error codes included).
6107	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6108	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6109	*
6110	* @param pVCpu Pointer to the VMCPU.
6111	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6112	* be out-of-sync. Make sure to update the required
6113	* fields before using them.
6114	* @param u64IntrInfo The VM-entry interruption-information field.
6115	* @param cbInstr The VM-entry instruction length in bytes (for
6116	* software interrupts, exceptions and privileged
6117	* software exceptions).
6118	* @param u32ErrCode The VM-entry exception error code.
6119	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6120	* @param puIntrState Pointer to the current guest interruptibility-state.
6121	* This interruptibility-state will be updated if
6122	* necessary. This cannot not be NULL.
6123	*
6124	* @remarks No-long-jump zone!!!
6125	* @remarks Requires CR0!
6126	*/
6127	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6128	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6129	{
6130	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6131	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6132	Assert(puIntrState);
6133	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6134
6135	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6136	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6137
6138	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6139	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6140	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6141
6142	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6143
6144	/* We require CR0 to check if the guest is in real-mode. */
6145	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6146	AssertRCReturn(rc, rc);
6147
6148	/*
6149	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6150	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6151	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6152	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6153	*/
6154	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6155	{
6156	PVM pVM = pVCpu->CTX_SUFF(pVM);
6157	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6158	{
6159	Assert(PDMVmmDevHeapIsEnabled(pVM));
6160	Assert(pVM->hm.s.vmx.pRealModeTSS);
6161
6162	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6163	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6164	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6165	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6166	AssertRCReturn(rc, rc);
6167	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6168
6169	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6170	const size_t cbIdtEntry = 4;
6171	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6172	{
6173	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6174	if (uVector == X86_XCPT_DF)
6175	return VINF_EM_RESET;
6176	else if (uVector == X86_XCPT_GP)
6177	{
6178	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6179	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6180	}
6181
6182	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6183	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6184	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6185	}
6186
6187	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6188	uint16_t uGuestIp = pMixedCtx->ip;
6189	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6190	{
6191	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6192	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6193	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6194	}
6195	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6196	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6197
6198	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6199	uint16_t offIdtEntry = 0;
6200	RTSEL selIdtEntry = 0;
6201	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6202	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6203	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6204	AssertRCReturn(rc, rc);
6205
6206	/* Construct the stack frame for the interrupt/exception handler. */
6207	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6208	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6209	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6210	AssertRCReturn(rc, rc);
6211
6212	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6213	if (rc == VINF_SUCCESS)
6214	{
6215	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6216	pMixedCtx->rip = offIdtEntry;
6217	pMixedCtx->cs.Sel = selIdtEntry;
6218	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6219	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6220	&& uVector == X86_XCPT_PF)
6221	{
6222	pMixedCtx->cr2 = GCPtrFaultAddress;
6223	}
6224	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6225	\| HM_CHANGED_GUEST_RIP
6226	\| HM_CHANGED_GUEST_RFLAGS
6227	\| HM_CHANGED_GUEST_RSP;
6228
6229	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6230	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6231	{
6232	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6233	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6234	Log(("Clearing inhibition due to STI.\n"));
6235	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6236	}
6237	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6238	}
6239	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6240	return rc;
6241	}
6242	else
6243	{
6244	/*
6245	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6246	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6247	*/
6248	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6249	}
6250	}
6251
6252	/* Validate. */
6253	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6254	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6255	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6256
6257	/* Inject. */
6258	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6259	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6260	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6261	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6262
6263	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6264	&& uVector == X86_XCPT_PF)
6265	{
6266	pMixedCtx->cr2 = GCPtrFaultAddress;
6267	}
6268	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6269
6270	AssertRCReturn(rc, rc);
6271	return rc;
6272	}
6273
6274
6275	/**
6276	* Enters the VT-x session.
6277	*
6278	* @returns VBox status code.
6279	* @param pVM Pointer to the VM.
6280	* @param pVCpu Pointer to the VMCPU.
6281	* @param pCpu Pointer to the CPU info struct.
6282	*/
6283	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6284	{
6285	AssertPtr(pVM);
6286	AssertPtr(pVCpu);
6287	Assert(pVM->hm.s.vmx.fSupported);
6288	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6289	NOREF(pCpu);
6290
6291	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6292
6293	/* Make sure we're in VMX root mode. */
6294	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6295	if (!(u32HostCR4 & X86_CR4_VMXE))
6296	{
6297	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6298	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6299	}
6300
6301	/* Load the active VMCS as the current one. */
6302	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6303	if (RT_FAILURE(rc))
6304	return rc;
6305
6306	/** @todo this will change with preemption hooks where can can VMRESUME as long
6307	* as we're no preempted. */
6308	pVCpu->hm.s.fResumeVM = false;
6309	return VINF_SUCCESS;
6310	}
6311
6312
6313	/**
6314	* Leaves the VT-x session.
6315	*
6316	* @returns VBox status code.
6317	* @param pVM Pointer to the VM.
6318	* @param pVCpu Pointer to the VMCPU.
6319	* @param pCtx Pointer to the guest-CPU context.
6320	*/
6321	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6322	{
6323	AssertPtr(pVCpu);
6324	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6325	NOREF(pVM);
6326	NOREF(pCtx);
6327
6328	/** @todo this will change with preemption hooks where we only VMCLEAR when
6329	* we are actually going to be preempted, not all the time like we
6330	* currently do. */
6331	/*
6332	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6333	* and mark the VMCS launch-state as "clear".
6334	*/
6335	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6336	return rc;
6337	}
6338
6339
6340	/**
6341	* Saves the host state in the VMCS host-state.
6342	* Sets up the VM-exit MSR-load area.
6343	*
6344	* The CPU state will be loaded from these fields on every successful VM-exit.
6345	*
6346	* @returns VBox status code.
6347	* @param pVM Pointer to the VM.
6348	* @param pVCpu Pointer to the VMCPU.
6349	*
6350	* @remarks No-long-jump zone!!!
6351	*/
6352	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6353	{
6354	AssertPtr(pVM);
6355	AssertPtr(pVCpu);
6356	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6357
6358	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6359
6360	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6361	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6362	return VINF_SUCCESS;
6363
6364	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6365	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6366
6367	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6368	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6369
6370	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6371	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6372
6373	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6374	return rc;
6375	}
6376
6377
6378	/**
6379	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6380	* loaded from these fields on every successful VM-entry.
6381	*
6382	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6383	* Sets up the VM-entry controls.
6384	* Sets up the appropriate VMX non-root function to execute guest code based on
6385	* the guest CPU mode.
6386	*
6387	* @returns VBox status code.
6388	* @param pVM Pointer to the VM.
6389	* @param pVCpu Pointer to the VMCPU.
6390	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6391	* out-of-sync. Make sure to update the required fields
6392	* before using them.
6393	*
6394	* @remarks No-long-jump zone!!!
6395	*/
6396	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6397	{
6398	AssertPtr(pVM);
6399	AssertPtr(pVCpu);
6400	AssertPtr(pMixedCtx);
6401	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6402
6403	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6404
6405	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6406
6407	/* Determine real-on-v86 mode. */
6408	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6409	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6410	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6411	{
6412	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6413	}
6414
6415	/*
6416	* Load the guest-state into the VMCS.
6417	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6418	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6419	*/
6420	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6421	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6422
6423	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6424	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6425
6426	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6427	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6428
6429	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6430	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6431
6432	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6433	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6434	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6435
6436	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6437	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6438
6439	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6440	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6441
6442	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6443	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6444
6445	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6446	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6447
6448	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6449	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6450
6451	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6452	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6453	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6454
6455	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6456	return rc;
6457	}
6458
6459
6460	/**
6461	* Does the preparations before executing guest code in VT-x.
6462	*
6463	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6464	* recompiler. We must be cautious what we do here regarding committing
6465	* guest-state information into the the VMCS assuming we assuredly execute the
6466	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6467	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6468	* that the recompiler can (and should) use them when it resumes guest
6469	* execution. Otherwise such operations must be done when we can no longer
6470	* exit to ring-3.
6471	*
6472	* @returns VBox status code (informational status codes included).
6473	* @retval VINF_SUCCESS if we can proceed with running the guest.
6474	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6475	* into the guest.
6476	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6477	*
6478	* @param pVM Pointer to the VM.
6479	* @param pVCpu Pointer to the VMCPU.
6480	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6481	* out-of-sync. Make sure to update the required fields
6482	* before using them.
6483	* @param pVmxTransient Pointer to the VMX transient structure.
6484	*
6485	* @remarks Called with preemption disabled.
6486	*/
6487	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6488	{
6489	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6490
6491	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6492	PGMRZDynMapFlushAutoSet(pVCpu);
6493	#endif
6494
6495	/* Check force flag actions that might require us to go back to ring-3. */
6496	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6497	if (rc != VINF_SUCCESS)
6498	return rc;
6499
6500	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6501	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6502	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6503	{
6504	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6505	RTGCPHYS GCPhysApicBase;
6506	GCPhysApicBase = pMixedCtx->msrApicBase;
6507	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6508
6509	/* Unalias any existing mapping. */
6510	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6511	AssertRCReturn(rc, rc);
6512
6513	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6514	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6515	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6516	AssertRCReturn(rc, rc);
6517
6518	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6519	}
6520
6521	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6522	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6523	pVmxTransient->uEFlags = ASMIntDisableFlags();
6524	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6525	{
6526	ASMSetFlags(pVmxTransient->uEFlags);
6527	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6528	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6529	return VINF_EM_RAW_INTERRUPT;
6530	}
6531	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6532	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6533	#endif
6534
6535	/*
6536	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6537	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6538	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6539	*/
6540	/** @todo Rework event evaluation and injection to be complete separate. */
6541	if (TRPMHasTrap(pVCpu))
6542	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6543
6544	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6545	AssertRCReturn(rc, rc);
6546	return rc;
6547	}
6548
6549
6550	/**
6551	* Prepares to run guest code in VT-x and we've committed to doing so. This
6552	* means there is no backing out to ring-3 or anywhere else at this
6553	* point.
6554	*
6555	* @param pVM Pointer to the VM.
6556	* @param pVCpu Pointer to the VMCPU.
6557	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6558	* out-of-sync. Make sure to update the required fields
6559	* before using them.
6560	* @param pVmxTransient Pointer to the VMX transient structure.
6561	*
6562	* @remarks Called with preemption disabled.
6563	* @remarks No-long-jump zone!!!
6564	*/
6565	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6566	{
6567	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6568	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6569
6570	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6571	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6572	pVmxTransient->uEFlags = ASMIntDisableFlags();
6573	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6574	#endif
6575
6576	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6577	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6578	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6579	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6580	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6581	#endif
6582	int rc = VINF_SUCCESS;
6583	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6584	{
6585	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6586	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6587	}
6588	else if (pVCpu->hm.s.fContextUseFlags)
6589	{
6590	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6591	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6592	}
6593	AssertRC(rc);
6594	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6595
6596	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6597	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6598	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6599
6600	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6601	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6602	{
6603	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6604	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6605	}
6606
6607	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6608	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6609	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6610
6611	/*
6612	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6613	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6614	*/
6615	if (pVM->hm.s.fTPRPatchingActive)
6616	{
6617	Assert(!CPUMIsGuestInLongMode(pVCpu));
6618
6619	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6620	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6621	AssertRC(rc);
6622
6623	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6624	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6625	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6626	see hmR0VmxLoadGuestApicState(). */
6627	}
6628
6629	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6630	/*
6631	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6632	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6633	*/
6634	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6635	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6636	{
6637	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6638	uint64_t u64HostTscAux = 0;
6639	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6640	AssertRC(rc2);
6641	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6642	}
6643	#endif
6644
6645	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6646	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6647	to start executing. */
6648	}
6649
6650
6651	/**
6652	* Performs some essential restoration of state after running guest code in
6653	* VT-x.
6654	*
6655	* @param pVM Pointer to the VM.
6656	* @param pVCpu Pointer to the VMCPU.
6657	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6658	* out-of-sync. Make sure to update the required fields
6659	* before using them.
6660	* @param pVmxTransient Pointer to the VMX transient structure.
6661	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6662	*
6663	* @remarks Called with interrupts disabled.
6664	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6665	* unconditionally when it is safe to do so.
6666	*/
6667	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6668	{
6669	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6670	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6671
6672	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6673	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6674	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6675	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6676	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6677
6678	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6679	{
6680	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6681	/* Restore host's TSC_AUX. */
6682	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6683	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6684	#endif
6685	/** @todo Find a way to fix hardcoding a guestimate. */
6686	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6687	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6688	}
6689
6690	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6691	Assert(!(ASMGetFlags() & X86_EFL_IF));
6692	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6693
6694	/* Restore the effects of TPR patching if any. */
6695	if (pVM->hm.s.fTPRPatchingActive)
6696	{
6697	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6698	AssertRC(rc);
6699	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6700	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6701	}
6702
6703	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6704	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6705
6706	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6707	uint32_t uExitReason;
6708	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6709	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6710	AssertRC(rc);
6711	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6712	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6713
6714	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6715	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6716
6717	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6718	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6719	{
6720	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6721	return;
6722	}
6723
6724	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6725	{
6726	/* Update the guest interruptibility-state from the VMCS. */
6727	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6728	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6729	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6730	AssertRC(rc);
6731	#endif
6732	/*
6733	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6734	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6735	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6736	*/
6737	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6738	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6739	{
6740	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6741	AssertRC(rc);
6742	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6743	}
6744	}
6745	}
6746
6747
6748	/**
6749	* Runs the guest code using VT-x.
6750	*
6751	* @returns VBox status code.
6752	* @param pVM Pointer to the VM.
6753	* @param pVCpu Pointer to the VMCPU.
6754	* @param pCtx Pointer to the guest-CPU context.
6755	*
6756	* @remarks Called with preemption disabled.
6757	*/
6758	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6759	{
6760	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6761	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6762
6763	VMXTRANSIENT VmxTransient;
6764	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6765	int rc = VERR_INTERNAL_ERROR_5;
6766	uint32_t cLoops = 0;
6767
6768	for (;; cLoops++)
6769	{
6770	Assert(!HMR0SuspendPending());
6771	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6772	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6773	(unsigned)RTMpCpuId(), cLoops));
6774
6775	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6776	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6777	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6778	if (rc != VINF_SUCCESS)
6779	break;
6780
6781	/*
6782	* No longjmps to ring-3 from this point on!!!
6783	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6784	* This also disables flushing of the R0-logger instance (if any).
6785	*/
6786	VMMRZCallRing3Disable(pVCpu);
6787	VMMRZCallRing3RemoveNotification(pVCpu);
6788	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6789
6790	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6791	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6792
6793	/*
6794	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6795	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6796	*/
6797	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6798	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6799	{
6800	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6801	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6802	return rc;
6803	}
6804
6805	/* Handle the VM-exit. */
6806	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6807	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6808	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6809	HMVMX_START_EXIT_DISPATCH_PROF();
6810	#ifdef HMVMX_USE_FUNCTION_TABLE
6811	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6812	#else
6813	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6814	#endif
6815	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6816	if (rc != VINF_SUCCESS)
6817	break;
6818	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6819	{
6820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6821	rc = VINF_EM_RAW_INTERRUPT;
6822	break;
6823	}
6824	}
6825
6826	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6827	if (rc == VERR_EM_INTERPRETER)
6828	rc = VINF_EM_RAW_EMULATE_INSTR;
6829	else if (rc == VINF_EM_RESET)
6830	rc = VINF_EM_TRIPLE_FAULT;
6831	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6832	return rc;
6833	}
6834
6835
6836	#ifndef HMVMX_USE_FUNCTION_TABLE
6837	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6838	{
6839	int rc;
6840	switch (rcReason)
6841	{
6842	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6853	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6854	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6856	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6857	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6858	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6859	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6860	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6861	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6862	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6863	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6864	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6865	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6866	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6867	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6868	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6869	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6870	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6871	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6872	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6873	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6874	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6875
6876	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6877	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6878	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6879	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6880	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6881	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6882	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6883	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6884	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6885
6886	case VMX_EXIT_VMCALL:
6887	case VMX_EXIT_VMCLEAR:
6888	case VMX_EXIT_VMLAUNCH:
6889	case VMX_EXIT_VMPTRLD:
6890	case VMX_EXIT_VMPTRST:
6891	case VMX_EXIT_VMREAD:
6892	case VMX_EXIT_VMRESUME:
6893	case VMX_EXIT_VMWRITE:
6894	case VMX_EXIT_VMXOFF:
6895	case VMX_EXIT_VMXON:
6896	case VMX_EXIT_INVEPT:
6897	case VMX_EXIT_INVVPID:
6898	case VMX_EXIT_VMFUNC:
6899	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6900	break;
6901	default:
6902	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6903	break;
6904	}
6905	return rc;
6906	}
6907	#endif
6908
6909	#ifdef DEBUG
6910	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6911	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6912	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6913	# define VMX_ASSERT_PREEMPT_CPUID() \
6914	do \
6915	{ \
6916	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6917	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6918	} while (0)
6919
6920	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6921	do { \
6922	AssertPtr(pVCpu); \
6923	AssertPtr(pMixedCtx); \
6924	AssertPtr(pVmxTransient); \
6925	Assert(pVmxTransient->fVMEntryFailed == false); \
6926	Assert(ASMIntAreEnabled()); \
6927	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6928	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6929	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6930	(unsigned)pVCpu->idCpu)); \
6931	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6932	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6933	VMX_ASSERT_PREEMPT_CPUID(); \
6934	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6935	} while (0)
6936	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6937	do { \
6938	LogFunc(("\n")); \
6939	} while(0)
6940	#else /* Release builds */
6941	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6942	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6943	#endif
6944
6945
6946	/**
6947	* Advances the guest RIP after reading it from the VMCS.
6948	*
6949	* @returns VBox status code.
6950	* @param pVCpu Pointer to the VMCPU.
6951	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6952	* out-of-sync. Make sure to update the required fields
6953	* before using them.
6954	* @param pVmxTransient Pointer to the VMX transient structure.
6955	*
6956	* @remarks No-long-jump zone!!!
6957	*/
6958	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6959	{
6960	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6961	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6962	AssertRCReturn(rc, rc);
6963
6964	pMixedCtx->rip += pVmxTransient->cbInstr;
6965	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6966	return rc;
6967	}
6968
6969
6970	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6971	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6972	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6973	/**
6974	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6975	*/
6976	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6977	{
6978	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6979	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6980	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6981	Assert(ASMIntAreEnabled());
6982	return VINF_SUCCESS;
6983	#else
6984	return VINF_EM_RAW_INTERRUPT;
6985	#endif
6986	}
6987
6988
6989	/**
6990	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6991	*/
6992	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6993	{
6994	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6995	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6996
6997	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6998	AssertRCReturn(rc, rc);
6999
7000	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7001	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
7002	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7003
7004	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7005	{
7006	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7007	return VINF_EM_RAW_INTERRUPT;
7008	}
7009
7010	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7011	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7012	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7013	{
7014	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7015	return VINF_SUCCESS;
7016	}
7017	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7018	{
7019	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7020	return rc;
7021	}
7022
7023	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7024	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7025	switch (uIntrType)
7026	{
7027	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7028	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7029	/* no break */
7030	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7031	{
7032	switch (uVector)
7033	{
7034	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7035	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7036	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7037	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7038	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7039	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7040	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7041	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7042	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7043	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7044	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7045	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7046	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7047	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7048	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7049	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7050	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7051	#endif
7052	default:
7053	{
7054	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7055	AssertRCReturn(rc, rc);
7056
7057	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7058	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7059	{
7060	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7061	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7062	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7063	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7064	AssertRCReturn(rc, rc);
7065	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7066	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7067	0 /* GCPtrFaultAddress */);
7068	AssertRCReturn(rc, rc);
7069	}
7070	else
7071	{
7072	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7073	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7074	}
7075	break;
7076	}
7077	}
7078	break;
7079	}
7080
7081	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7082	default:
7083	{
7084	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7085	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7086	break;
7087	}
7088	}
7089	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7090	return rc;
7091	}
7092
7093
7094	/**
7095	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7096	*/
7097	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7098	{
7099	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7100
7101	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7102	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7103	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7104	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7105	AssertRCReturn(rc, rc);
7106
7107	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7109	return VINF_SUCCESS;
7110	}
7111
7112
7113	/**
7114	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7115	*/
7116	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7117	{
7118	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7119	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7120	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7121	}
7122
7123
7124	/**
7125	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7126	*/
7127	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7128	{
7129	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7131	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7132	}
7133
7134
7135	/**
7136	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7137	*/
7138	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7139	{
7140	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7141	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7142	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7143	}
7144
7145
7146	/**
7147	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7148	*/
7149	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7150	{
7151	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7152	PVM pVM = pVCpu->CTX_SUFF(pVM);
7153	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7154	if (RT_LIKELY(rc == VINF_SUCCESS))
7155	{
7156	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7157	Assert(pVmxTransient->cbInstr == 2);
7158	}
7159	else
7160	{
7161	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7162	rc = VERR_EM_INTERPRETER;
7163	}
7164	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7165	return rc;
7166	}
7167
7168
7169	/**
7170	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7171	*/
7172	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7173	{
7174	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7175	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7176	AssertRCReturn(rc, rc);
7177
7178	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7179	return VINF_EM_RAW_EMULATE_INSTR;
7180
7181	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7182	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7183	}
7184
7185
7186	/**
7187	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7188	*/
7189	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7190	{
7191	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7192	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7193	AssertRCReturn(rc, rc);
7194
7195	PVM pVM = pVCpu->CTX_SUFF(pVM);
7196	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7197	if (RT_LIKELY(rc == VINF_SUCCESS))
7198	{
7199	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7200	Assert(pVmxTransient->cbInstr == 2);
7201	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7202	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7203	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7204	}
7205	else
7206	{
7207	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7208	rc = VERR_EM_INTERPRETER;
7209	}
7210	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7211	return rc;
7212	}
7213
7214
7215	/**
7216	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7217	*/
7218	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7219	{
7220	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7221	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7222	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7223	AssertRCReturn(rc, rc);
7224
7225	PVM pVM = pVCpu->CTX_SUFF(pVM);
7226	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7227	if (RT_LIKELY(rc == VINF_SUCCESS))
7228	{
7229	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7230	Assert(pVmxTransient->cbInstr == 3);
7231	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7232	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7233	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7234	}
7235	else
7236	{
7237	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7238	rc = VERR_EM_INTERPRETER;
7239	}
7240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7241	return rc;
7242	}
7243
7244
7245	/**
7246	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7247	*/
7248	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7249	{
7250	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7251	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7252	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7253	AssertRCReturn(rc, rc);
7254
7255	PVM pVM = pVCpu->CTX_SUFF(pVM);
7256	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7257	if (RT_LIKELY(rc == VINF_SUCCESS))
7258	{
7259	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7260	Assert(pVmxTransient->cbInstr == 2);
7261	}
7262	else
7263	{
7264	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7265	rc = VERR_EM_INTERPRETER;
7266	}
7267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7268	return rc;
7269	}
7270
7271
7272	/**
7273	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7274	*/
7275	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7276	{
7277	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7278	PVM pVM = pVCpu->CTX_SUFF(pVM);
7279	Assert(!pVM->hm.s.fNestedPaging);
7280
7281	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7282	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7283	AssertRCReturn(rc, rc);
7284
7285	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7286	rc = VBOXSTRICTRC_VAL(rc2);
7287	if (RT_LIKELY(rc == VINF_SUCCESS))
7288	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7289	else
7290	{
7291	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7292	pVmxTransient->uExitQualification, rc));
7293	}
7294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7295	return rc;
7296	}
7297
7298
7299	/**
7300	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7301	*/
7302	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7303	{
7304	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7305	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7306	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7307	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7308	AssertRCReturn(rc, rc);
7309
7310	PVM pVM = pVCpu->CTX_SUFF(pVM);
7311	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7312	if (RT_LIKELY(rc == VINF_SUCCESS))
7313	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7314	else
7315	{
7316	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7317	rc = VERR_EM_INTERPRETER;
7318	}
7319	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7320	return rc;
7321	}
7322
7323
7324	/**
7325	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7326	*/
7327	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7328	{
7329	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7330	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7331	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7332	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7333	AssertRCReturn(rc, rc);
7334
7335	PVM pVM = pVCpu->CTX_SUFF(pVM);
7336	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7337	rc = VBOXSTRICTRC_VAL(rc2);
7338	if (RT_LIKELY( rc == VINF_SUCCESS
7339	\|\| rc == VINF_EM_HALT))
7340	{
7341	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7342	AssertRCReturn(rc3, rc3);
7343
7344	if ( rc == VINF_EM_HALT
7345	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7346	{
7347	rc = VINF_SUCCESS;
7348	}
7349	}
7350	else
7351	{
7352	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7353	rc = VERR_EM_INTERPRETER;
7354	}
7355	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7356	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7357	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7358	return rc;
7359	}
7360
7361
7362	/**
7363	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7364	*/
7365	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7366	{
7367	/*
7368	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7369	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7370	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7371	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7372	*/
7373	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7374	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7375	}
7376
7377
7378	/**
7379	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7380	*/
7381	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7382	{
7383	/*
7384	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7385	* root operation. If we get there there is something funny going on.
7386	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7387	*/
7388	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7389	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7390	}
7391
7392
7393	/**
7394	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7395	*/
7396	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7397	{
7398	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7399	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7400	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7401	}
7402
7403
7404	/**
7405	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7406	*/
7407	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7408	{
7409	/*
7410	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7411	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7412	* See Intel spec. 25.3 "Other Causes of VM-exits".
7413	*/
7414	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7415	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7416	}
7417
7418
7419	/**
7420	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7421	* VM-exit.
7422	*/
7423	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7424	{
7425	/*
7426	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7427	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7428	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7429	*/
7430	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7431	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7432	}
7433
7434
7435	/**
7436	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7437	* VM-exit.
7438	*/
7439	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7440	{
7441	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7442	return VINF_EM_RESET;
7443	}
7444
7445
7446	/**
7447	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7448	*/
7449	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7450	{
7451	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7452	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7453	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7454	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7455	AssertRCReturn(rc, rc);
7456
7457	pMixedCtx->rip++;
7458	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7459	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7460	rc = VINF_SUCCESS;
7461	else
7462	rc = VINF_EM_HALT;
7463
7464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7465	return rc;
7466	}
7467
7468
7469	/**
7470	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7471	*/
7472	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7473	{
7474	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7475	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7476	return VINF_SUCCESS;
7477	}
7478
7479
7480	/**
7481	* VM-exit handler for expiry of the VMX preemption timer.
7482	*/
7483	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7484	{
7485	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7486
7487	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7488	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7489
7490	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7491	PVM pVM = pVCpu->CTX_SUFF(pVM);
7492	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7494	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7495	}
7496
7497
7498	/**
7499	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7500	*/
7501	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7502	{
7503	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7504	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7505	/** @todo check if XSETBV is supported by the recompiler. */
7506	return VERR_EM_INTERPRETER;
7507	}
7508
7509
7510	/**
7511	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7512	*/
7513	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7514	{
7515	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7516	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7517	/** @todo implement EMInterpretInvpcid() */
7518	return VERR_EM_INTERPRETER;
7519	}
7520
7521
7522	/**
7523	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7524	* Error VM-exit.
7525	*/
7526	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7527	{
7528	uint32_t uIntrState;
7529	HMVMXHCUINTREG uHCReg;
7530	uint64_t u64Val;
7531	uint32_t u32Val;
7532
7533	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7534	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7535	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7536	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7537	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7538	AssertRCReturn(rc, rc);
7539
7540	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7541	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7542	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7543	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7544
7545	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7546	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7547	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7548	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7549	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7550	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7551	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7552	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7553	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7554	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7555	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7556	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7557
7558	PVM pVM = pVCpu->CTX_SUFF(pVM);
7559	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7560
7561	return VERR_VMX_INVALID_GUEST_STATE;
7562	}
7563
7564
7565	/**
7566	* VM-exit handler for VM-entry failure due to an MSR-load
7567	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7568	*/
7569	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7570	{
7571	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7572	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7573	}
7574
7575
7576	/**
7577	* VM-exit handler for VM-entry failure due to a machine-check event
7578	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7579	*/
7580	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7581	{
7582	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7583	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7584	}
7585
7586
7587	/**
7588	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7589	* theory.
7590	*/
7591	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7592	{
7593	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7594	return VERR_VMX_UNDEFINED_EXIT_CODE;
7595	}
7596
7597
7598	/**
7599	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7600	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7601	* Conditional VM-exit.
7602	*/
7603	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7604	{
7605	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7606	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7607	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7608	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7609	return VERR_EM_INTERPRETER;
7610	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7611	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7612	}
7613
7614
7615	/**
7616	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7617	*/
7618	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7619	{
7620	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7621	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7623	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7624	return VERR_EM_INTERPRETER;
7625	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7626	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7627	}
7628
7629
7630	/**
7631	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7632	*/
7633	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7634	{
7635	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7636	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7637	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7638	AssertRCReturn(rc, rc);
7639	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7640	AssertRCReturn(rc, rc);
7641	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7642	AssertRCReturn(rc, rc);
7643
7644	PVM pVM = pVCpu->CTX_SUFF(pVM);
7645	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7646	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7647	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7649
7650	if (RT_LIKELY(rc == VINF_SUCCESS))
7651	{
7652	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7653	Assert(pVmxTransient->cbInstr == 2);
7654	}
7655	return rc;
7656	}
7657
7658
7659	/**
7660	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7661	*/
7662	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7663	{
7664	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7665	PVM pVM = pVCpu->CTX_SUFF(pVM);
7666	int rc = VINF_SUCCESS;
7667
7668	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7669	if ( pVM->hm.s.fTPRPatchingActive
7670	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7671	{
7672	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7673	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7674	{
7675	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7676	AssertRC(rc);
7677	}
7678
7679	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7680	Assert(pVmxTransient->cbInstr == 2);
7681	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7682	return VINF_SUCCESS;
7683	}
7684
7685	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7686	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7687	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7688	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7689	AssertRCReturn(rc, rc);
7690	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7691
7692	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7693	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7694	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7695
7696	if (RT_LIKELY(rc == VINF_SUCCESS))
7697	{
7698	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7699
7700	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7701	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7702	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7703	{
7704	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7705	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7706	}
7707	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7708	{
7709	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7710	AssertRCReturn(rc, rc);
7711	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7712	}
7713	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7714	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7715
7716	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7717	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7718	{
7719	switch (pMixedCtx->ecx)
7720	{
7721	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7722	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7723	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7724	case MSR_K8_FS_BASE: /* no break */
7725	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7726	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7727	}
7728	}
7729	#ifdef VBOX_STRICT
7730	else
7731	{
7732	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7733	switch (pMixedCtx->ecx)
7734	{
7735	case MSR_IA32_SYSENTER_CS:
7736	case MSR_IA32_SYSENTER_EIP:
7737	case MSR_IA32_SYSENTER_ESP:
7738	case MSR_K8_FS_BASE:
7739	case MSR_K8_GS_BASE:
7740	{
7741	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7742	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7743	}
7744
7745	case MSR_K8_LSTAR:
7746	case MSR_K6_STAR:
7747	case MSR_K8_SF_MASK:
7748	case MSR_K8_TSC_AUX:
7749	case MSR_K8_KERNEL_GS_BASE:
7750	{
7751	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7752	pMixedCtx->ecx));
7753	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7754	}
7755	}
7756	}
7757	#endif /* VBOX_STRICT */
7758	}
7759	return rc;
7760	}
7761
7762
7763	/**
7764	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7765	*/
7766	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7767	{
7768	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7769	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7770	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7771	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7772	return VERR_EM_INTERPRETER;
7773	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7774	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7775	}
7776
7777
7778	/**
7779	* VM-exit handler for when the TPR value is lowered below the specified
7780	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7781	*/
7782	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7783	{
7784	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7785	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7786
7787	/*
7788	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7789	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7790	* resume guest execution.
7791	*/
7792	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7793	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7794	return VINF_SUCCESS;
7795	}
7796
7797
7798	/**
7799	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7800	* VM-exit.
7801	*
7802	* @retval VINF_SUCCESS when guest execution can continue.
7803	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7804	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7805	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7806	* recompiler.
7807	*/
7808	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7809	{
7810	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7811	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7812	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7813	AssertRCReturn(rc, rc);
7814
7815	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7816	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7817	PVM pVM = pVCpu->CTX_SUFF(pVM);
7818	switch (uAccessType)
7819	{
7820	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7821	{
7822	#if 0
7823	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7824	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7825	#else
7826	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7827	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7828	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7829	#endif
7830	AssertRCReturn(rc, rc);
7831
7832	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7833	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7834	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7835	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7836
7837	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7838	{
7839	case 0: /* CR0 */
7840	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7841	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7842	break;
7843	case 2: /* C2 **/
7844	/* Nothing to do here, CR2 it's not part of the VMCS. */
7845	break;
7846	case 3: /* CR3 */
7847	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7848	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7849	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7850	break;
7851	case 4: /* CR4 */
7852	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7853	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7854	break;
7855	case 8: /* CR8 */
7856	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7857	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7858	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7859	break;
7860	default:
7861	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7862	break;
7863	}
7864
7865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7866	break;
7867	}
7868
7869	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7870	{
7871	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7872	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7873	AssertRCReturn(rc, rc);
7874	Assert( !pVM->hm.s.fNestedPaging
7875	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7876	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7877
7878	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7879	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7880	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7881
7882	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7883	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7884	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7885	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7886	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7887	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7888	break;
7889	}
7890
7891	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7892	{
7893	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7894	AssertRCReturn(rc, rc);
7895	rc = EMInterpretCLTS(pVM, pVCpu);
7896	AssertRCReturn(rc, rc);
7897	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7899	Log(("CRX CLTS write rc=%d\n", rc));
7900	break;
7901	}
7902
7903	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7904	{
7905	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7906	AssertRCReturn(rc, rc);
7907	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7908	if (RT_LIKELY(rc == VINF_SUCCESS))
7909	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7911	Log(("CRX LMSW write rc=%d\n", rc));
7912	break;
7913	}
7914
7915	default:
7916	{
7917	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7918	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7919	}
7920	}
7921
7922	/* Validate possible error codes. */
7923	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7924	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7925	if (RT_SUCCESS(rc))
7926	{
7927	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7928	AssertRCReturn(rc2, rc2);
7929	}
7930
7931	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7932	return rc;
7933	}
7934
7935
7936	/**
7937	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7938	* VM-exit.
7939	*/
7940	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7941	{
7942	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7943	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7944
7945	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7946	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7947	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7948	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7949	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7950	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7951	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7952	AssertRCReturn(rc, rc);
7953	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7954
7955	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7956	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7957	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7958	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7959	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7960	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7961	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7962
7963	/* I/O operation lookup arrays. */
7964	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7965	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7966
7967	const uint32_t cbSize = s_aIOSize[uIOWidth];
7968	const uint32_t cbInstr = pVmxTransient->cbInstr;
7969	PVM pVM = pVCpu->CTX_SUFF(pVM);
7970	if (fIOString)
7971	{
7972	/* INS/OUTS - I/O String instruction. */
7973	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7974	/** @todo for now manually disassemble later optimize by getting the fields from
7975	* the VMCS. */
7976	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7977	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7978	* segment prefix info. */
7979	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7980	if (RT_SUCCESS(rc))
7981	{
7982	if (fIOWrite)
7983	{
7984	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7985	(DISCPUMODE)pDis->uAddrMode, cbSize);
7986	rc = VBOXSTRICTRC_VAL(rc2);
7987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7988	}
7989	else
7990	{
7991	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7992	(DISCPUMODE)pDis->uAddrMode, cbSize);
7993	rc = VBOXSTRICTRC_VAL(rc2);
7994	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7995	}
7996	}
7997	else
7998	{
7999	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8000	rc = VINF_EM_RAW_EMULATE_INSTR;
8001	}
8002	}
8003	else
8004	{
8005	/* IN/OUT - I/O instruction. */
8006	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8007	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8008	if (fIOWrite)
8009	{
8010	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8011	rc = VBOXSTRICTRC_VAL(rc2);
8012	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8013	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8014	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8015	}
8016	else
8017	{
8018	uint32_t u32Result = 0;
8019	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8020	rc = VBOXSTRICTRC_VAL(rc2);
8021	if (IOM_SUCCESS(rc))
8022	{
8023	/* Save result of I/O IN instr. in AL/AX/EAX. */
8024	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8025	}
8026	else if (rc == VINF_IOM_R3_IOPORT_READ)
8027	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8028	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8029	}
8030	}
8031
8032	if (IOM_SUCCESS(rc))
8033	{
8034	pMixedCtx->rip += cbInstr;
8035	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8036	if (RT_LIKELY(rc == VINF_SUCCESS))
8037	{
8038	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8039	AssertRCReturn(rc, rc);
8040
8041	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8042	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8043	{
8044	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8045	for (unsigned i = 0; i < 4; i++)
8046	{
8047	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8048	if ( ( uIOPort >= pMixedCtx->dr[i]
8049	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8050	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8051	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8052	{
8053	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8054	uint64_t uDR6 = ASMGetDR6();
8055
8056	/* Clear all breakpoint status flags and set the one we just hit. */
8057	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8058	uDR6 \|= (uint64_t)RT_BIT(i);
8059
8060	/*
8061	* Note: AMD64 Architecture Programmer's Manual 13.1:
8062	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8063	* be cleared by software after the contents have been read.
8064	*/
8065	ASMSetDR6(uDR6);
8066
8067	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8068	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8069
8070	/* Paranoia. */
8071	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8072	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8073	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8074
8075	/* Resync DR7 */
8076	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8077	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8078
8079	/* Set #DB to be injected into the VM and continue guest execution. */
8080	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8081	break;
8082	}
8083	}
8084	}
8085	}
8086	}
8087
8088	#ifdef DEBUG
8089	if (rc == VINF_IOM_R3_IOPORT_READ)
8090	Assert(!fIOWrite);
8091	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8092	Assert(fIOWrite);
8093	else
8094	{
8095	AssertMsg( RT_FAILURE(rc)
8096	\|\| rc == VINF_SUCCESS
8097	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8098	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8099	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8100	}
8101	#endif
8102
8103	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8104	return rc;
8105	}
8106
8107
8108	/**
8109	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8110	* VM-exit.
8111	*/
8112	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8113	{
8114	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8115
8116	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8117	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8118	AssertRCReturn(rc, rc);
8119	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8120	{
8121	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8122	AssertRCReturn(rc, rc);
8123	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8124	{
8125	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8126	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8127	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8128	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8129	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8130	{
8131	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8132	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8133
8134	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8135	Assert(!pVCpu->hm.s.Event.fPending);
8136	pVCpu->hm.s.Event.fPending = true;
8137	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8138	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8139	AssertRCReturn(rc, rc);
8140	if (fErrorCodeValid)
8141	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8142	else
8143	pVCpu->hm.s.Event.u32ErrCode = 0;
8144	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8145	&& uVector == X86_XCPT_PF)
8146	{
8147	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8148	}
8149	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8150	}
8151	}
8152	}
8153	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8154	* emulation. */
8155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8156	return VERR_EM_INTERPRETER;
8157	}
8158
8159
8160	/**
8161	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8162	*/
8163	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8164	{
8165	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8166	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8167	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8168	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8169	AssertRCReturn(rc, rc);
8170	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8171	return VINF_EM_DBG_STOP;
8172	}
8173
8174
8175	/**
8176	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8177	*/
8178	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8179	{
8180	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8181
8182	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8183	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8184	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8185	return VINF_SUCCESS;
8186	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8187	return rc;
8188
8189	#if 0
8190	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8191	* just sync the whole thing. */
8192	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8193	#else
8194	/* Aggressive state sync. for now. */
8195	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8196	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8197	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8198	#endif
8199	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8200	AssertRCReturn(rc, rc);
8201
8202	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8203	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8204	switch (uAccessType)
8205	{
8206	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8207	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8208	{
8209	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8210	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8211	{
8212	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8213	}
8214
8215	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8216	GCPhys &= PAGE_BASE_GC_MASK;
8217	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8218	PVM pVM = pVCpu->CTX_SUFF(pVM);
8219	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8220	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8221
8222	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8223	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8224	CPUMCTX2CORE(pMixedCtx), GCPhys);
8225	rc = VBOXSTRICTRC_VAL(rc2);
8226	Log(("ApicAccess rc=%d\n", rc));
8227	if ( rc == VINF_SUCCESS
8228	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8229	\|\| rc == VERR_PAGE_NOT_PRESENT)
8230	{
8231	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8232	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8233	rc = VINF_SUCCESS;
8234	}
8235	break;
8236	}
8237
8238	default:
8239	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8240	rc = VINF_EM_RAW_EMULATE_INSTR;
8241	break;
8242	}
8243
8244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8245	return rc;
8246	}
8247
8248
8249	/**
8250	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8251	* VM-exit.
8252	*/
8253	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8254	{
8255	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8256
8257	/* We should -not- get this VM-exit if the guest is debugging. */
8258	if (CPUMIsGuestDebugStateActive(pVCpu))
8259	{
8260	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8261	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8262	}
8263
8264	int rc = VERR_INTERNAL_ERROR_5;
8265	if ( !DBGFIsStepping(pVCpu)
8266	&& !CPUMIsHyperDebugStateActive(pVCpu))
8267	{
8268	/* Don't intercept MOV DRx. */
8269	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8270	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8271	AssertRCReturn(rc, rc);
8272
8273	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8274	PVM pVM = pVCpu->CTX_SUFF(pVM);
8275	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8276	AssertRC(rc);
8277	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8278
8279	#ifdef VBOX_WITH_STATISTICS
8280	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8281	AssertRCReturn(rc, rc);
8282	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8284	else
8285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8286	#endif
8287	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8288	return VINF_SUCCESS;
8289	}
8290
8291	/*
8292	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8293	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8294	*/
8295	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8296	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8297	AssertRCReturn(rc, rc);
8298
8299	PVM pVM = pVCpu->CTX_SUFF(pVM);
8300	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8301	{
8302	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8303	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8304	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8305	if (RT_SUCCESS(rc))
8306	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8308	}
8309	else
8310	{
8311	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8312	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8313	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8315	}
8316
8317	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8318	if (RT_SUCCESS(rc))
8319	{
8320	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8321	AssertRCReturn(rc2, rc2);
8322	}
8323	return rc;
8324	}
8325
8326
8327	/**
8328	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8329	* Conditional VM-exit.
8330	*/
8331	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8332	{
8333	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8334	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8335
8336	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8337	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8338	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8339	return VINF_SUCCESS;
8340	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8341	return rc;
8342
8343	RTGCPHYS GCPhys = 0;
8344	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8345
8346	#if 0
8347	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8348	#else
8349	/* Aggressive state sync. for now. */
8350	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8351	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8352	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8353	#endif
8354	AssertRCReturn(rc, rc);
8355
8356	/*
8357	* If we succeed, resume guest execution.
8358	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8359	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8360	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8361	* weird case. See @bugref{6043}.
8362	*/
8363	PVM pVM = pVCpu->CTX_SUFF(pVM);
8364	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8365	rc = VBOXSTRICTRC_VAL(rc2);
8366	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8367	if ( rc == VINF_SUCCESS
8368	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8369	\|\| rc == VERR_PAGE_NOT_PRESENT)
8370	{
8371	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8372	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8373	return VINF_SUCCESS;
8374	}
8375	return rc;
8376	}
8377
8378
8379	/**
8380	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8381	* VM-exit.
8382	*/
8383	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8384	{
8385	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8386	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8387
8388	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8389	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8390	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8391	return VINF_SUCCESS;
8392	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8393	return rc;
8394
8395	RTGCPHYS GCPhys = 0;
8396	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8397	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8398	#if 0
8399	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8400	#else
8401	/* Aggressive state sync. for now. */
8402	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8403	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8404	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8405	#endif
8406	AssertRCReturn(rc, rc);
8407
8408	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8409	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8410
8411	RTGCUINT uErrorCode = 0;
8412	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8413	uErrorCode \|= X86_TRAP_PF_ID;
8414	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8415	uErrorCode \|= X86_TRAP_PF_RW;
8416	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8417	uErrorCode \|= X86_TRAP_PF_P;
8418
8419	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8420
8421	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8422	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8423
8424	/* Handle the pagefault trap for the nested shadow table. */
8425	PVM pVM = pVCpu->CTX_SUFF(pVM);
8426	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8427	TRPMResetTrap(pVCpu);
8428
8429	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8430	if ( rc == VINF_SUCCESS
8431	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8432	\|\| rc == VERR_PAGE_NOT_PRESENT)
8433	{
8434	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8436	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8437	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8438	return VINF_SUCCESS;
8439	}
8440
8441	Log(("EPT return to ring-3 rc=%d\n"));
8442	return rc;
8443	}
8444
8445
8446	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8447	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8448	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8449	/**
8450	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8451	*/
8452	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8453	{
8454	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8456
8457	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8458	AssertRCReturn(rc, rc);
8459
8460	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8461	{
8462	/* Old-style FPU error reporting needs some extra work. */
8463	/** @todo don't fall back to the recompiler, but do it manually. */
8464	return VERR_EM_INTERPRETER;
8465	}
8466	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8467	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8468	return rc;
8469	}
8470
8471
8472	/**
8473	* VM-exit exception handler for #BP (Breakpoint exception).
8474	*/
8475	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8476	{
8477	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8478	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8479
8480	/** @todo Try optimize this by not saving the entire guest state unless
8481	* really needed. */
8482	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8483	AssertRCReturn(rc, rc);
8484
8485	PVM pVM = pVCpu->CTX_SUFF(pVM);
8486	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8487	if (rc == VINF_EM_RAW_GUEST_TRAP)
8488	{
8489	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8490	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8491	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8492	AssertRCReturn(rc, rc);
8493
8494	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8495	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8496	}
8497
8498	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8499	return rc;
8500	}
8501
8502
8503	/**
8504	* VM-exit exception handler for #DB (Debug exception).
8505	*/
8506	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8507	{
8508	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8509	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8510
8511	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8512	AssertRCReturn(rc, rc);
8513
8514	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8515	uint64_t uDR6 = X86_DR6_INIT_VAL;
8516	uDR6 \|= (pVmxTransient->uExitQualification
8517	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8518	PVM pVM = pVCpu->CTX_SUFF(pVM);
8519	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8520	if (rc == VINF_EM_RAW_GUEST_TRAP)
8521	{
8522	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8523	pMixedCtx->dr[6] = uDR6;
8524
8525	if (CPUMIsGuestDebugStateActive(pVCpu))
8526	ASMSetDR6(pMixedCtx->dr[6]);
8527
8528	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8529
8530	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8531	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8532
8533	/* Paranoia. */
8534	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8535	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8536	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8537
8538	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8539	AssertRCReturn(rc,rc);
8540
8541	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8542	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8543	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8544	AssertRCReturn(rc2, rc2);
8545	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8546	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8547	rc = VINF_SUCCESS;
8548	}
8549
8550	return rc;
8551	}
8552
8553
8554	/**
8555	* VM-exit exception handler for #NM (Device-not-available exception: floating
8556	* point exception).
8557	*/
8558	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8559	{
8560	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8561
8562	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8563	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8564	#endif
8565
8566	/* We require CR0 and EFER. EFER is always up-to-date. */
8567	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8568	AssertRCReturn(rc, rc);
8569
8570	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8571	PVM pVM = pVCpu->CTX_SUFF(pVM);
8572	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8573	if (rc == VINF_SUCCESS)
8574	{
8575	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8576	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8578	return VINF_SUCCESS;
8579	}
8580
8581	/* Forward #NM to the guest. */
8582	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8583	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8584	AssertRCReturn(rc, rc);
8585	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8586	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8588	return rc;
8589	}
8590
8591
8592	/**
8593	* VM-exit exception handler for #GP (General-protection exception).
8594	*
8595	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8596	*/
8597	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8598	{
8599	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8601
8602	int rc = VERR_INTERNAL_ERROR_5;
8603	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8604	{
8605	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8606	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8607	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8608	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8609	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8610	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8611	AssertRCReturn(rc, rc);
8612	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8613	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8614	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8615	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8616	return rc;
8617	#else
8618	/* We don't intercept #GP. */
8619	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8620	return VERR_VMX_UNEXPECTED_EXCEPTION;
8621	#endif
8622	}
8623
8624	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8625	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8626
8627	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8628	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8629	AssertRCReturn(rc, rc);
8630
8631	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8632	uint32_t cbOp = 0;
8633	PVM pVM = pVCpu->CTX_SUFF(pVM);
8634	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8635	if (RT_SUCCESS(rc))
8636	{
8637	rc = VINF_SUCCESS;
8638	Assert(cbOp == pDis->cbInstr);
8639	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8640	switch (pDis->pCurInstr->uOpcode)
8641	{
8642	case OP_CLI:
8643	pMixedCtx->eflags.Bits.u1IF = 0;
8644	pMixedCtx->rip += pDis->cbInstr;
8645	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8647	break;
8648
8649	case OP_STI:
8650	pMixedCtx->eflags.Bits.u1IF = 1;
8651	pMixedCtx->rip += pDis->cbInstr;
8652	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8653	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8654	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8656	break;
8657
8658	case OP_HLT:
8659	rc = VINF_EM_HALT;
8660	pMixedCtx->rip += pDis->cbInstr;
8661	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8663	break;
8664
8665	case OP_POPF:
8666	{
8667	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8668	uint32_t cbParm = 0;
8669	uint32_t uMask = 0;
8670	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8671	{
8672	cbParm = 4;
8673	uMask = 0xffffffff;
8674	}
8675	else
8676	{
8677	cbParm = 2;
8678	uMask = 0xffff;
8679	}
8680
8681	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8682	RTGCPTR GCPtrStack = 0;
8683	X86EFLAGS uEflags;
8684	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8685	&GCPtrStack);
8686	if (RT_SUCCESS(rc))
8687	{
8688	Assert(sizeof(uEflags.u32) >= cbParm);
8689	uEflags.u32 = 0;
8690	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8691	}
8692	if (RT_FAILURE(rc))
8693	{
8694	rc = VERR_EM_INTERPRETER;
8695	break;
8696	}
8697	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8698	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8699	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8700	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8701	pMixedCtx->eflags.Bits.u1RF = 0;
8702	pMixedCtx->esp += cbParm;
8703	pMixedCtx->esp &= uMask;
8704	pMixedCtx->rip += pDis->cbInstr;
8705	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8707	break;
8708	}
8709
8710	case OP_PUSHF:
8711	{
8712	uint32_t cbParm = 0;
8713	uint32_t uMask = 0;
8714	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8715	{
8716	cbParm = 4;
8717	uMask = 0xffffffff;
8718	}
8719	else
8720	{
8721	cbParm = 2;
8722	uMask = 0xffff;
8723	}
8724
8725	/* Get the stack pointer & push the contents of eflags onto the stack. */
8726	RTGCPTR GCPtrStack = 0;
8727	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8728	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8729	if (RT_FAILURE(rc))
8730	{
8731	rc = VERR_EM_INTERPRETER;
8732	break;
8733	}
8734	X86EFLAGS uEflags;
8735	uEflags = pMixedCtx->eflags;
8736	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8737	uEflags.Bits.u1RF = 0;
8738	uEflags.Bits.u1VM = 0;
8739
8740	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8741	if (RT_FAILURE(rc))
8742	{
8743	rc = VERR_EM_INTERPRETER;
8744	break;
8745	}
8746	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8747	pMixedCtx->esp -= cbParm;
8748	pMixedCtx->esp &= uMask;
8749	pMixedCtx->rip += pDis->cbInstr;
8750	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8752	break;
8753	}
8754
8755	case OP_IRET:
8756	{
8757	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8758	* instruction reference. */
8759	RTGCPTR GCPtrStack = 0;
8760	uint32_t uMask = 0xffff;
8761	uint16_t aIretFrame[3];
8762	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8763	{
8764	rc = VERR_EM_INTERPRETER;
8765	break;
8766	}
8767	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8768	&GCPtrStack);
8769	if (RT_SUCCESS(rc))
8770	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8771	if (RT_FAILURE(rc))
8772	{
8773	rc = VERR_EM_INTERPRETER;
8774	break;
8775	}
8776	pMixedCtx->eip = 0;
8777	pMixedCtx->ip = aIretFrame[0];
8778	pMixedCtx->cs.Sel = aIretFrame[1];
8779	pMixedCtx->cs.ValidSel = aIretFrame[1];
8780	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8781	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8782	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8783	pMixedCtx->sp += sizeof(aIretFrame);
8784	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8785	\| HM_CHANGED_GUEST_RFLAGS;
8786	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8787	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8788	break;
8789	}
8790
8791	case OP_INT:
8792	{
8793	uint16_t uVector = pDis->Param1.uValue & 0xff;
8794	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8796	break;
8797	}
8798
8799	case OP_INTO:
8800	{
8801	if (pMixedCtx->eflags.Bits.u1OF)
8802	{
8803	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8805	}
8806	break;
8807	}
8808
8809	default:
8810	{
8811	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8812	EMCODETYPE_SUPERVISOR);
8813	rc = VBOXSTRICTRC_VAL(rc2);
8814	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8815	Log(("#GP rc=%Rrc\n", rc));
8816	break;
8817	}
8818	}
8819	}
8820	else
8821	rc = VERR_EM_INTERPRETER;
8822
8823	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8824	("#GP Unexpected rc=%Rrc\n", rc));
8825	return rc;
8826	}
8827
8828
8829	/**
8830	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8831	* the exception reported in the VMX transient structure back into the VM.
8832	*
8833	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8834	* up-to-date.
8835	*/
8836	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8837	{
8838	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8839
8840	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8841	hmR0VmxCheckExitDueToEventDelivery(). */
8842	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8843	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8844	AssertRCReturn(rc, rc);
8845	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8846	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8847	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8848	return VINF_SUCCESS;
8849	}
8850
8851
8852	/**
8853	* VM-exit exception handler for #PF (Page-fault exception).
8854	*/
8855	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8856	{
8857	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8858	PVM pVM = pVCpu->CTX_SUFF(pVM);
8859	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8860	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8861	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8862	AssertRCReturn(rc, rc);
8863
8864	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8865	if (pVM->hm.s.fNestedPaging)
8866	{
8867	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8868	{
8869	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8870	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8871	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8872	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8873	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8874	}
8875	else
8876	{
8877	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8878	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8879	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8880	Log(("Pending #DF due to vectoring #PF. NP\n"));
8881	}
8882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8883	return rc;
8884	}
8885	#else
8886	Assert(!pVM->hm.s.fNestedPaging);
8887	#endif
8888
8889	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8890	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8891	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8892	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8893	AssertRCReturn(rc, rc);
8894	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8895	if ( pVM->hm.s.fTRPPatchingAllowed
8896	&& pVM->hm.s.pGuestPatchMem
8897	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8898	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8899	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8900	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8901	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8902	{
8903	RTGCPHYS GCPhys;
8904	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8905	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8906	if ( rc == VINF_SUCCESS
8907	&& GCPhys == GCPhysApicBase)
8908	{
8909	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8910	AssertRCReturn(rc, rc);
8911
8912	/* Only attempt to patch the instruction once. */
8913	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8914	if (!pPatch)
8915	return VINF_EM_HM_PATCH_TPR_INSTR;
8916	}
8917	}
8918	#endif
8919
8920	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8921	AssertRCReturn(rc, rc);
8922
8923	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8924	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8925
8926	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8927	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8928	(RTGCPTR)pVmxTransient->uExitQualification);
8929
8930	Log(("#PF: rc=%Rrc\n", rc));
8931	if (rc == VINF_SUCCESS)
8932	{
8933	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8934	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8935	* memory? We don't update the whole state here... */
8936	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8937	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8938	TRPMResetTrap(pVCpu);
8939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8940	return rc;
8941	}
8942	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8943	{
8944	if (!pVmxTransient->fVectoringPF)
8945	{
8946	/* It's a guest page fault and needs to be reflected to the guest. */
8947	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8948	TRPMResetTrap(pVCpu);
8949	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8950	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8951	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8952	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8953	}
8954	else
8955	{
8956	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8957	TRPMResetTrap(pVCpu);
8958	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8959	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8960	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8961	}
8962
8963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8964	return VINF_SUCCESS;
8965	}
8966
8967	TRPMResetTrap(pVCpu);
8968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8969	return rc;
8970	}
8971

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45908

Download in other formats: