HMVMXR0.cpp@ 49284

Last change on this file since 49284 was 49284, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Don't forget to load guest debug state when not intercepting #DB and Mov DRx.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 461.4 KB

Line
1	/* $Id: HMVMXR0.cpp 49284 2013-10-25 09:45:30Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HMVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iem.h>
32	#include <VBox/vmm/iom.h>
33	#include <VBox/vmm/selm.h>
34	#include <VBox/vmm/tm.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#ifdef DEBUG_ramshankar
39	#define HMVMX_SAVE_FULL_GUEST_STATE
40	#define HMVMX_SYNC_FULL_GUEST_STATE
41	#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43	#define HMVMX_ALWAYS_TRAP_PF
44	#define HMVMX_ALWAYS_SWAP_FPU_STATE
45	#define HMVMX_ALWAYS_FLUSH_TLB
46	#endif
47
48
49	/*******************************************************************************
50	* Defined Constants And Macros *
51	*******************************************************************************/
52	#if defined(RT_ARCH_AMD64)
53	# define HMVMX_IS_64BIT_HOST_MODE() (true)
54	typedef RTHCUINTREG HMVMXHCUINTREG;
55	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56	extern "C" uint32_t g_fVMXIs64bitHost;
57	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
58	typedef uint64_t HMVMXHCUINTREG;
59	#else
60	# define HMVMX_IS_64BIT_HOST_MODE() (false)
61	typedef RTHCUINTREG HMVMXHCUINTREG;
62	#endif
63
64	/** Use the function table. */
65	#define HMVMX_USE_FUNCTION_TABLE
66
67	/**
68	* The maximum number of MSRs we are willing to swap during a world-switch.
69	* Intel claims 512/check capability MSR, we don't want to do anywhere close
70	* to that. See Intel spec. 24.7.2 "VM-Exit Controls for MSRs"
71	*
72	* Bump this count as and when required, there's no backward compatibility
73	* requirement.
74	*/
75	#define HMVMX_MAX_SWAP_MSR_COUNT 5
76
77	/** Determine which tagged-TLB flush handler to use. */
78	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
79	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
80	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
81	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
82
83	/** @name Updated-guest-state flags.
84	* @{ */
85	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
86	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
87	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
88	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
89	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
90	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
91	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
92	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
93	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
94	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
95	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
96	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
97	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
98	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
99	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
100	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
101	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
102	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
103	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
104	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
105	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
106	\| HMVMX_UPDATED_GUEST_RSP \
107	\| HMVMX_UPDATED_GUEST_RFLAGS \
108	\| HMVMX_UPDATED_GUEST_CR0 \
109	\| HMVMX_UPDATED_GUEST_CR3 \
110	\| HMVMX_UPDATED_GUEST_CR4 \
111	\| HMVMX_UPDATED_GUEST_GDTR \
112	\| HMVMX_UPDATED_GUEST_IDTR \
113	\| HMVMX_UPDATED_GUEST_LDTR \
114	\| HMVMX_UPDATED_GUEST_TR \
115	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
116	\| HMVMX_UPDATED_GUEST_DEBUG \
117	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
118	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
119	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
120	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
121	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
122	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
123	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
124	\| HMVMX_UPDATED_GUEST_APIC_STATE)
125	/** @} */
126
127	/** @name
128	* Flags to skip redundant reads of some common VMCS fields that are not part of
129	* the guest-CPU state but are in the transient structure.
130	*/
131	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
132	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
133	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
134	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
135	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
136	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
137	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
138	/** @} */
139
140	/** @name
141	* States of the VMCS.
142	*
143	* This does not reflect all possible VMCS states but currently only those
144	* needed for maintaining the VMCS consistently even when thread-context hooks
145	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
146	*/
147	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
148	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
149	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
150	/** @} */
151
152	/**
153	* Exception bitmap mask for real-mode guests (real-on-v86).
154	*
155	* We need to intercept all exceptions manually (except #PF). #NM is also
156	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
157	* even in real-mode if we have Nested Paging support.
158	*/
159	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
160	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
161	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
162	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
163	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
164	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
165	\| RT_BIT(X86_XCPT_XF))
166
167	/**
168	* Exception bitmap mask for all contributory exceptions.
169	*
170	* Page fault is deliberately excluded here as it's conditional as to whether
171	* it's contributory or benign. Page faults are handled separately.
172	*/
173	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
174	\| RT_BIT(X86_XCPT_DE))
175
176	/** Maximum VM-instruction error number. */
177	#define HMVMX_INSTR_ERROR_MAX 28
178
179	/** Profiling macro. */
180	#ifdef HM_PROFILE_EXIT_DISPATCH
181	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
182	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
183	#else
184	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
185	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
186	#endif
187
188	/** Assert that preemption is disabled or covered by thread-context hooks. */
189	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
191
192	/** Assert that we haven't migrated CPUs when thread-context hooks are not
193	* used. */
194	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
195	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
196	("Illegal migration! Entered on CPU %u Current %u\n", \
197	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
198
199	/** Helper macro for VM-exit handlers called unexpectedly. */
200	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
201	do { \
202	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
203	return VERR_VMX_UNEXPECTED_EXIT; \
204	} while (0)
205
206
207	/*******************************************************************************
208	* Structures and Typedefs *
209	*******************************************************************************/
210	/**
211	* VMX transient state.
212	*
213	* A state structure for holding miscellaneous information across
214	* VMX non-root operation and restored after the transition.
215	*/
216	typedef struct VMXTRANSIENT
217	{
218	/** The host's rflags/eflags. */
219	RTCCUINTREG uEflags;
220	#if HC_ARCH_BITS == 32
221	uint32_t u32Alignment0;
222	#endif
223	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
224	uint64_t u64LStarMsr;
225	/** The guest's TPR value used for TPR shadowing. */
226	uint8_t u8GuestTpr;
227	/** Alignment. */
228	uint8_t abAlignment0[7];
229
230	/** The basic VM-exit reason. */
231	uint16_t uExitReason;
232	/** Alignment. */
233	uint16_t u16Alignment0;
234	/** The VM-exit interruption error code. */
235	uint32_t uExitIntErrorCode;
236	/** The VM-exit exit qualification. */
237	uint64_t uExitQualification;
238
239	/** The VM-exit interruption-information field. */
240	uint32_t uExitIntInfo;
241	/** The VM-exit instruction-length field. */
242	uint32_t cbInstr;
243	/** The VM-exit instruction-information field. */
244	union
245	{
246	/** Plain unsigned int representation. */
247	uint32_t u;
248	/** INS and OUTS information. */
249	struct
250	{
251	uint32_t u6Reserved0 : 7;
252	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253	uint32_t u3AddrSize : 3;
254	uint32_t u5Reserved1 : 5;
255	/** The segment register (X86_SREG_XXX). */
256	uint32_t iSegReg : 3;
257	uint32_t uReserved2 : 14;
258	} StrIo;
259	} ExitInstrInfo;
260	/** Whether the VM-entry failed or not. */
261	bool fVMEntryFailed;
262	/** Alignment. */
263	uint8_t abAlignment1[3];
264
265	/** The VM-entry interruption-information field. */
266	uint32_t uEntryIntInfo;
267	/** The VM-entry exception error code field. */
268	uint32_t uEntryXcptErrorCode;
269	/** The VM-entry instruction length field. */
270	uint32_t cbEntryInstr;
271
272	/** IDT-vectoring information field. */
273	uint32_t uIdtVectoringInfo;
274	/** IDT-vectoring error code. */
275	uint32_t uIdtVectoringErrorCode;
276
277	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
278	uint32_t fVmcsFieldsRead;
279
280	/** Whether the guest FPU was active at the time of VM-exit. */
281	bool fWasGuestFPUStateActive;
282	/** Whether the guest debug state was active at the time of VM-exit. */
283	bool fWasGuestDebugStateActive;
284	/** Whether the hyper debug state was active at the time of VM-exit. */
285	bool fWasHyperDebugStateActive;
286	/** Whether TSC-offsetting should be setup before VM-entry. */
287	bool fUpdateTscOffsettingAndPreemptTimer;
288	/** Whether the VM-exit was caused by a page-fault during delivery of a
289	* contributory exception or a page-fault. */
290	bool fVectoringPF;
291	} VMXTRANSIENT;
292	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
296	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297	/** Pointer to VMX transient state. */
298	typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301	/**
302	* MSR-bitmap read permissions.
303	*/
304	typedef enum VMXMSREXITREAD
305	{
306	/** Reading this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_READ
310	} VMXMSREXITREAD;
311
312	/**
313	* MSR-bitmap write permissions.
314	*/
315	typedef enum VMXMSREXITWRITE
316	{
317	/** Writing to this MSR causes a VM-exit. */
318	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a VM-exit. */
320	VMXMSREXIT_PASSTHRU_WRITE
321	} VMXMSREXITWRITE;
322
323	/**
324	* VMX VM-exit handler.
325	*
326	* @returns VBox status code.
327	* @param pVCpu Pointer to the VMCPU.
328	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
329	* out-of-sync. Make sure to update the required
330	* fields before using them.
331	* @param pVmxTransient Pointer to the VMX-transient structure.
332	*/
333	#ifndef HMVMX_USE_FUNCTION_TABLE
334	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335	#else
336	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337	/** Pointer to VM-exit handler. */
338	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
339	#endif
340
341
342	/*******************************************************************************
343	* Internal Functions *
344	*******************************************************************************/
345	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
346	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
347	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
348	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
350	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
351	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
352	#endif
353	#ifndef HMVMX_USE_FUNCTION_TABLE
354	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
355	# define HMVMX_EXIT_DECL static int
356	#else
357	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
358	#endif
359
360	/** @name VM-exit handlers.
361	* @{
362	*/
363	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
364	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
365	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
366	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
367	static FNVMXEXITHANDLER hmR0VmxExitSipi;
368	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
369	static FNVMXEXITHANDLER hmR0VmxExitSmi;
370	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
371	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
372	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
373	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
374	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
375	static FNVMXEXITHANDLER hmR0VmxExitHlt;
376	static FNVMXEXITHANDLER hmR0VmxExitInvd;
377	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
378	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
379	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
380	static FNVMXEXITHANDLER hmR0VmxExitRsm;
381	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
382	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
383	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
384	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
385	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
386	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
387	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
388	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
389	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
390	static FNVMXEXITHANDLER hmR0VmxExitMwait;
391	static FNVMXEXITHANDLER hmR0VmxExitMtf;
392	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
393	static FNVMXEXITHANDLER hmR0VmxExitPause;
394	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
395	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
396	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
397	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
398	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
399	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
400	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
401	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
402	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
403	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
404	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
405	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
406	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
407	/** @} */
408
409	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
417
418	/*******************************************************************************
419	* Global Variables *
420	*******************************************************************************/
421	#ifdef HMVMX_USE_FUNCTION_TABLE
422
423	/**
424	* VMX_EXIT dispatch table.
425	*/
426	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
427	{
428	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
429	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
430	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
431	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
432	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
433	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
434	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
435	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
436	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
437	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
438	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
439	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
440	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
441	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
442	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
443	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
444	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
445	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
446	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
447	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
448	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
449	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
450	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
451	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
452	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
453	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
454	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
455	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
456	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
457	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
458	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
459	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
460	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
461	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
462	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
463	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
464	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
465	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
466	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
468	/* 40 UNDEFINED */ hmR0VmxExitPause,
469	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
470	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
471	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
472	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
473	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
474	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
475	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
476	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
477	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
478	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
479	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
480	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
481	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
482	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
483	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
484	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
485	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
486	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
487	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
488	};
489	#endif /* HMVMX_USE_FUNCTION_TABLE */
490
491	#ifdef VBOX_STRICT
492	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
493	{
494	/* 0 */ "(Not Used)",
495	/* 1 */ "VMCALL executed in VMX root operation.",
496	/* 2 */ "VMCLEAR with invalid physical address.",
497	/* 3 */ "VMCLEAR with VMXON pointer.",
498	/* 4 */ "VMLAUNCH with non-clear VMCS.",
499	/* 5 */ "VMRESUME with non-launched VMCS.",
500	/* 6 */ "VMRESUME after VMXOFF",
501	/* 7 */ "VM entry with invalid control fields.",
502	/* 8 */ "VM entry with invalid host state fields.",
503	/* 9 */ "VMPTRLD with invalid physical address.",
504	/* 10 */ "VMPTRLD with VMXON pointer.",
505	/* 11 */ "VMPTRLD with incorrect revision identifier.",
506	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
507	/* 13 */ "VMWRITE to read-only VMCS component.",
508	/* 14 */ "(Not Used)",
509	/* 15 */ "VMXON executed in VMX root operation.",
510	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
511	/* 17 */ "VM entry with non-launched executing VMCS.",
512	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
513	/* 19 */ "VMCALL with non-clear VMCS.",
514	/* 20 */ "VMCALL with invalid VM-exit control fields.",
515	/* 21 */ "(Not Used)",
516	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
517	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
518	/* 24 */ "VMCALL with invalid SMM-monitor features.",
519	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
520	/* 26 */ "VM entry with events blocked by MOV SS.",
521	/* 27 */ "(Not Used)",
522	/* 28 */ "Invalid operand to INVEPT/INVVPID."
523	};
524	#endif /* VBOX_STRICT */
525
526
527
528	/**
529	* Updates the VM's last error record. If there was a VMX instruction error,
530	* reads the error data from the VMCS and updates VCPU's last error record as
531	* well.
532	*
533	* @param pVM Pointer to the VM.
534	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
535	* VERR_VMX_UNABLE_TO_START_VM or
536	* VERR_VMX_INVALID_VMCS_FIELD).
537	* @param rc The error code.
538	*/
539	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
540	{
541	AssertPtr(pVM);
542	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
543	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
544	{
545	AssertPtrReturnVoid(pVCpu);
546	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
547	}
548	pVM->hm.s.lLastError = rc;
549	}
550
551
552	/**
553	* Reads the VM-entry interruption-information field from the VMCS into the VMX
554	* transient structure.
555	*
556	* @returns VBox status code.
557	* @param pVmxTransient Pointer to the VMX transient structure.
558	*
559	* @remarks No-long-jump zone!!!
560	*/
561	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
562	{
563	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
564	AssertRCReturn(rc, rc);
565	return VINF_SUCCESS;
566	}
567
568
569	/**
570	* Reads the VM-entry exception error code field from the VMCS into
571	* the VMX transient structure.
572	*
573	* @returns VBox status code.
574	* @param pVmxTransient Pointer to the VMX transient structure.
575	*
576	* @remarks No-long-jump zone!!!
577	*/
578	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579	{
580	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
581	AssertRCReturn(rc, rc);
582	return VINF_SUCCESS;
583	}
584
585
586	/**
587	* Reads the VM-entry exception error code field from the VMCS into
588	* the VMX transient structure.
589	*
590	* @returns VBox status code.
591	* @param pVCpu Pointer to the VMCPU.
592	* @param pVmxTransient Pointer to the VMX transient structure.
593	*
594	* @remarks No-long-jump zone!!!
595	*/
596	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
597	{
598	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
599	AssertRCReturn(rc, rc);
600	return VINF_SUCCESS;
601	}
602
603
604	/**
605	* Reads the VM-exit interruption-information field from the VMCS into the VMX
606	* transient structure.
607	*
608	* @returns VBox status code.
609	* @param pVCpu Pointer to the VMCPU.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*/
612	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
613	{
614	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
617	AssertRCReturn(rc, rc);
618	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
619	}
620	return VINF_SUCCESS;
621	}
622
623
624	/**
625	* Reads the VM-exit interruption error code from the VMCS into the VMX
626	* transient structure.
627	*
628	* @returns VBox status code.
629	* @param pVCpu Pointer to the VMCPU.
630	* @param pVmxTransient Pointer to the VMX transient structure.
631	*/
632	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
633	{
634	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635	{
636	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637	AssertRCReturn(rc, rc);
638	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639	}
640	return VINF_SUCCESS;
641	}
642
643
644	/**
645	* Reads the VM-exit instruction length field from the VMCS into the VMX
646	* transient structure.
647	*
648	* @returns VBox status code.
649	* @param pVCpu Pointer to the VMCPU.
650	* @param pVmxTransient Pointer to the VMX transient structure.
651	*/
652	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
653	{
654	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655	{
656	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657	AssertRCReturn(rc, rc);
658	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659	}
660	return VINF_SUCCESS;
661	}
662
663
664	/**
665	* Reads the VM-exit instruction-information field from the VMCS into
666	* the VMX transient structure.
667	*
668	* @returns VBox status code.
669	* @param pVCpu The cross context per CPU structure.
670	* @param pVmxTransient Pointer to the VMX transient structure.
671	*/
672	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
673	{
674	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675	{
676	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677	AssertRCReturn(rc, rc);
678	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679	}
680	return VINF_SUCCESS;
681	}
682
683
684	/**
685	* Reads the exit qualification from the VMCS into the VMX transient structure.
686	*
687	* @returns VBox status code.
688	* @param pVCpu Pointer to the VMCPU.
689	* @param pVmxTransient Pointer to the VMX transient structure.
690	*/
691	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
692	{
693	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
694	{
695	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
696	AssertRCReturn(rc, rc);
697	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
698	}
699	return VINF_SUCCESS;
700	}
701
702
703	/**
704	* Reads the IDT-vectoring information field from the VMCS into the VMX
705	* transient structure.
706	*
707	* @returns VBox status code.
708	* @param pVmxTransient Pointer to the VMX transient structure.
709	*
710	* @remarks No-long-jump zone!!!
711	*/
712	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
713	{
714	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
715	{
716	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
717	AssertRCReturn(rc, rc);
718	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
719	}
720	return VINF_SUCCESS;
721	}
722
723
724	/**
725	* Reads the IDT-vectoring error code from the VMCS into the VMX
726	* transient structure.
727	*
728	* @returns VBox status code.
729	* @param pVmxTransient Pointer to the VMX transient structure.
730	*/
731	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
732	{
733	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
734	{
735	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
736	AssertRCReturn(rc, rc);
737	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
738	}
739	return VINF_SUCCESS;
740	}
741
742
743	/**
744	* Enters VMX root mode operation on the current CPU.
745	*
746	* @returns VBox status code.
747	* @param pVM Pointer to the VM (optional, can be NULL, after
748	* a resume).
749	* @param HCPhysCpuPage Physical address of the VMXON region.
750	* @param pvCpuPage Pointer to the VMXON region.
751	*/
752	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
753	{
754	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
755	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
756	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758	if (pVM)
759	{
760	/* Write the VMCS revision dword to the VMXON region. */
761	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
762	}
763
764	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
765	RTCCUINTREG uEflags = ASMIntDisableFlags();
766
767	/* Enable the VMX bit in CR4 if necessary. */
768	RTCCUINTREG uCr4 = ASMGetCR4();
769	if (!(uCr4 & X86_CR4_VMXE))
770	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
771
772	/* Enter VMX root mode. */
773	int rc = VMXEnable(HCPhysCpuPage);
774	if (RT_FAILURE(rc))
775	ASMSetCR4(uCr4);
776
777	/* Restore interrupts. */
778	ASMSetFlags(uEflags);
779	return rc;
780	}
781
782
783	/**
784	* Exits VMX root mode operation on the current CPU.
785	*
786	* @returns VBox status code.
787	*/
788	static int hmR0VmxLeaveRootMode(void)
789	{
790	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
791
792	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
793	RTCCUINTREG uEflags = ASMIntDisableFlags();
794
795	/* If we're for some reason not in VMX root mode, then don't leave it. */
796	RTCCUINTREG uHostCR4 = ASMGetCR4();
797
798	int rc;
799	if (uHostCR4 & X86_CR4_VMXE)
800	{
801	/* Exit VMX root mode and clear the VMX bit in CR4. */
802	VMXDisable();
803	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
804	rc = VINF_SUCCESS;
805	}
806	else
807	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
808
809	/* Restore interrupts. */
810	ASMSetFlags(uEflags);
811	return rc;
812	}
813
814
815	/**
816	* Allocates and maps one physically contiguous page. The allocated page is
817	* zero'd out. (Used by various VT-x structures).
818	*
819	* @returns IPRT status code.
820	* @param pMemObj Pointer to the ring-0 memory object.
821	* @param ppVirt Where to store the virtual address of the
822	* allocation.
823	* @param pPhys Where to store the physical address of the
824	* allocation.
825	*/
826	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
827	{
828	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
829	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
830	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
831
832	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
833	if (RT_FAILURE(rc))
834	return rc;
835	ppVirt = RTR0MemObjAddress(pMemObj);
836	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
837	ASMMemZero32(*ppVirt, PAGE_SIZE);
838	return VINF_SUCCESS;
839	}
840
841
842	/**
843	* Frees and unmaps an allocated physical page.
844	*
845	* @param pMemObj Pointer to the ring-0 memory object.
846	* @param ppVirt Where to re-initialize the virtual address of
847	* allocation as 0.
848	* @param pHCPhys Where to re-initialize the physical address of the
849	* allocation as 0.
850	*/
851	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
852	{
853	AssertPtr(pMemObj);
854	AssertPtr(ppVirt);
855	AssertPtr(pHCPhys);
856	if (*pMemObj != NIL_RTR0MEMOBJ)
857	{
858	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
859	AssertRC(rc);
860	*pMemObj = NIL_RTR0MEMOBJ;
861	*ppVirt = 0;
862	*pHCPhys = 0;
863	}
864	}
865
866
867	/**
868	* Worker function to free VT-x related structures.
869	*
870	* @returns IPRT status code.
871	* @param pVM Pointer to the VM.
872	*/
873	static void hmR0VmxStructsFree(PVM pVM)
874	{
875	for (VMCPUID i = 0; i < pVM->cCpus; i++)
876	{
877	PVMCPU pVCpu = &pVM->aCpus[i];
878	AssertPtr(pVCpu);
879
880	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
881	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
882	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
883	#endif
884
885	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
886	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
887
888	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
889	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
890	}
891
892	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
893	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
894	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
895	#endif
896	}
897
898
899	/**
900	* Worker function to allocate VT-x related VM structures.
901	*
902	* @returns IPRT status code.
903	* @param pVM Pointer to the VM.
904	*/
905	static int hmR0VmxStructsAlloc(PVM pVM)
906	{
907	/*
908	* Initialize members up-front so we can cleanup properly on allocation failure.
909	*/
910	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
911	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
912	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
913	pVM->hm.s.vmx.HCPhys##a_Name = 0;
914
915	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
916	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
919
920	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
921	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
922	#endif
923	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
924
925	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
926	for (VMCPUID i = 0; i < pVM->cCpus; i++)
927	{
928	PVMCPU pVCpu = &pVM->aCpus[i];
929	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
930	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
931	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
932	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
933	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935	#endif
936	}
937	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
938	#undef VMXLOCAL_INIT_VM_MEMOBJ
939
940	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
941	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
942	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
943	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
944
945	/*
946	* Allocate all the VT-x structures.
947	*/
948	int rc = VINF_SUCCESS;
949	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951	if (RT_FAILURE(rc))
952	goto cleanup;
953	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
954	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
955	#endif
956
957	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
958	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
959	{
960	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
961	&pVM->hm.s.vmx.HCPhysApicAccess);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964	}
965
966	/*
967	* Initialize per-VCPU VT-x structures.
968	*/
969	for (VMCPUID i = 0; i < pVM->cCpus; i++)
970	{
971	PVMCPU pVCpu = &pVM->aCpus[i];
972	AssertPtr(pVCpu);
973
974	/* Allocate the VM control structure (VMCS). */
975	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
976	if (RT_FAILURE(rc))
977	goto cleanup;
978
979	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
980	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
981	{
982	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
983	&pVCpu->hm.s.vmx.HCPhysVirtApic);
984	if (RT_FAILURE(rc))
985	goto cleanup;
986	}
987
988	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
989	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
990	{
991	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
992	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
993	if (RT_FAILURE(rc))
994	goto cleanup;
995	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
996	}
997
998	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
999	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1000	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1001	if (RT_FAILURE(rc))
1002	goto cleanup;
1003
1004	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1005	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1006	if (RT_FAILURE(rc))
1007	goto cleanup;
1008	#endif
1009	}
1010
1011	return VINF_SUCCESS;
1012
1013	cleanup:
1014	hmR0VmxStructsFree(pVM);
1015	return rc;
1016	}
1017
1018
1019	/**
1020	* Does global VT-x initialization (called during module initialization).
1021	*
1022	* @returns VBox status code.
1023	*/
1024	VMMR0DECL(int) VMXR0GlobalInit(void)
1025	{
1026	#ifdef HMVMX_USE_FUNCTION_TABLE
1027	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1028	# ifdef VBOX_STRICT
1029	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1030	Assert(g_apfnVMExitHandlers[i]);
1031	# endif
1032	#endif
1033	return VINF_SUCCESS;
1034	}
1035
1036
1037	/**
1038	* Does global VT-x termination (called during module termination).
1039	*/
1040	VMMR0DECL(void) VMXR0GlobalTerm()
1041	{
1042	/* Nothing to do currently. */
1043	}
1044
1045
1046	/**
1047	* Sets up and activates VT-x on the current CPU.
1048	*
1049	* @returns VBox status code.
1050	* @param pCpu Pointer to the global CPU info struct.
1051	* @param pVM Pointer to the VM (can be NULL after a host resume
1052	* operation).
1053	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1054	* fEnabledByHost is true).
1055	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1056	* @a fEnabledByHost is true).
1057	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1058	* enable VT-x on the host.
1059	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1060	*/
1061	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1062	void *pvMsrs)
1063	{
1064	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1065	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1066	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067
1068	/* Enable VT-x if it's not already enabled by the host. */
1069	if (!fEnabledByHost)
1070	{
1071	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1072	if (RT_FAILURE(rc))
1073	return rc;
1074	}
1075
1076	/*
1077	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1078	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1079	*/
1080	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1081	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1082	{
1083	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1084	pCpu->fFlushAsidBeforeUse = false;
1085	}
1086	else
1087	pCpu->fFlushAsidBeforeUse = true;
1088
1089	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1090	++pCpu->cTlbFlushes;
1091
1092	return VINF_SUCCESS;
1093	}
1094
1095
1096	/**
1097	* Deactivates VT-x on the current CPU.
1098	*
1099	* @returns VBox status code.
1100	* @param pCpu Pointer to the global CPU info struct.
1101	* @param pvCpuPage Pointer to the VMXON region.
1102	* @param HCPhysCpuPage Physical address of the VMXON region.
1103	*
1104	* @remarks This function should never be called when SUPR0EnableVTx() or
1105	* similar was used to enable VT-x on the host.
1106	*/
1107	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1108	{
1109	NOREF(pCpu);
1110	NOREF(pvCpuPage);
1111	NOREF(HCPhysCpuPage);
1112
1113	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1114	return hmR0VmxLeaveRootMode();
1115	}
1116
1117
1118	/**
1119	* Sets the permission bits for the specified MSR in the MSR bitmap.
1120	*
1121	* @param pVCpu Pointer to the VMCPU.
1122	* @param uMSR The MSR value.
1123	* @param enmRead Whether reading this MSR causes a VM-exit.
1124	* @param enmWrite Whether writing this MSR causes a VM-exit.
1125	*/
1126	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1127	{
1128	int32_t iBit;
1129	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1130
1131	/*
1132	* Layout:
1133	* 0x000 - 0x3ff - Low MSR read bits
1134	* 0x400 - 0x7ff - High MSR read bits
1135	* 0x800 - 0xbff - Low MSR write bits
1136	* 0xc00 - 0xfff - High MSR write bits
1137	*/
1138	if (uMsr <= 0x00001FFF)
1139	iBit = uMsr;
1140	else if ( uMsr >= 0xC0000000
1141	&& uMsr <= 0xC0001FFF)
1142	{
1143	iBit = (uMsr - 0xC0000000);
1144	pbMsrBitmap += 0x400;
1145	}
1146	else
1147	{
1148	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1149	return;
1150	}
1151
1152	Assert(iBit <= 0x1fff);
1153	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154	ASMBitSet(pbMsrBitmap, iBit);
1155	else
1156	ASMBitClear(pbMsrBitmap, iBit);
1157
1158	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160	else
1161	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162	}
1163
1164
1165	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1166	/**
1167	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1168	* area.
1169	*
1170	* @returns VBox status code.
1171	* @param pVCpu Pointer to the VMCPU.
1172	* @param cMsrs The number of MSRs.
1173	*/
1174	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1175	{
1176	/* Update the VCPU's copy of the guest MSR count. */
1177	pVCpu->hm.s.vmx.cGuestMsrs = cMsrs;
1178
1179	/* Update number of guest MSRs to load/store across the world-switch. */
1180	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1181	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1182
1183	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1184	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1185	return VINF_SUCCESS;
1186	}
1187
1188
1189	/**
1190	* Adds a guest/host MSR pair to be swapped during the world-switch as
1191	* part of the auto-load/store MSR area in the VMCS.
1192	*
1193	* @returns VBox status code.
1194	* @param pVCpu Pointer to the VMCPU.
1195	* @param uMsr The MSR.
1196	* @param uGuestMsr Value of the guest MSR.
1197	* @param uHostMsr Value of the host MSR.
1198	*/
1199	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, uint64_t uHostMsrValue)
1200	{
1201	AssertMsg(HMVMX_MAX_SWAP_MSR_COUNT < MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc),
1202	("MSR swap count exceeded. Cpu reports %#RX32, our limit %#RX32\n",
1203	MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc), HMVMX_MAX_SWAP_MSR_COUNT));
1204
1205	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1206	uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1207	uint32_t i;
1208	for (i = 0; i < cGuestMsrs; i++)
1209	{
1210	if (pGuestMsr->u32Msr == uMsr)
1211	break;
1212	pGuestMsr++;
1213	}
1214
1215	AssertReturn(i < HMVMX_MAX_SWAP_MSR_COUNT, VERR_HM_MSR_SWAP_COUNT_EXCEEDED);
1216	if (i == cGuestMsrs)
1217	{
1218	++cGuestMsrs;
1219	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1220	if (RT_UNLIKELY(cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc)))
1221	{
1222	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
1223	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1224	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1225	}
1226
1227	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1228	AssertRCReturn(rc, rc);
1229	}
1230
1231	/* Update the MSR values in the auto-load/store MSR area. */
1232	pGuestMsr->u32Msr = uMsr;
1233	pGuestMsr->u64Value = uGuestMsrValue;
1234
1235	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1236	pHostMsr += i;
1237	pHostMsr->u32Msr = uMsr;
1238	pHostMsr->u64Value = uHostMsrValue;
1239
1240	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1241	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
1242	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1243	return VINF_SUCCESS;
1244	}
1245
1246
1247	/**
1248	* Removes a guest/shost MSR pair to be swapped during the world-switch from the
1249	* auto-load/store MSR area in the VMCS.
1250	*
1251	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1252	* area.
1253	*
1254	* @returns VBox status code.
1255	* @param pVCpu Pointer to the VMCPU.
1256	* @param uMsr The MSR.
1257	*/
1258	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1259	{
1260	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1261	uint32_t cGuestMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1262	uint32_t i;
1263	for (i = 0; i < cGuestMsrs; i++)
1264	{
1265	/* Find the MSR. */
1266	if (pGuestMsr->u32Msr == uMsr)
1267	{
1268	/* If it's the last MSR, simply reduce the count. */
1269	if (i == cGuestMsrs - 1)
1270	{
1271	--cGuestMsrs;
1272	break;
1273	}
1274
1275	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1276	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1277	pLastGuestMsr += cGuestMsrs;
1278	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1279	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1280
1281	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1283	pLastHostMsr += cGuestMsrs;
1284	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1285	pHostMsr->u64Value = pLastHostMsr->u64Value;
1286	--cGuestMsrs;
1287	break;
1288	}
1289	pGuestMsr++;
1290	}
1291
1292	/* Update the VMCS if the count changed (meaning the MSR was found). */
1293	if (cGuestMsrs != pVCpu->hm.s.vmx.cGuestMsrs)
1294	{
1295	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cGuestMsrs);
1296	AssertRCReturn(rc, rc);
1297	}
1298
1299	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1300	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1301	return VINF_SUCCESS;
1302	}
1303
1304
1305	/**
1306	* Updates the value of a host MSR in the auto-load/store area in the VMCS.
1307	*
1308	* @returns VBox status code.
1309	* @param pVCpu Pointer to the VMCPU.
1310	* @param uMsr The MSR.
1311	*/
1312	static int hmR0VmxUpdateAutoLoadStoreHostMsr(PVMCPU pVCpu, uint32_t uMsr)
1313	{
1314	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1315	uint32_t cMsrs = pVCpu->hm.s.vmx.cGuestMsrs;
1316
1317	for (uint32_t i = 0; i < cMsrs; i++)
1318	{
1319	if (pHostMsr->u32Msr == uMsr)
1320	{
1321	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1322	return VINF_SUCCESS;
1323	}
1324	}
1325
1326	return VERR_NOT_FOUND;
1327	}
1328
1329	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1330
1331
1332	/**
1333	* Flushes the TLB using EPT.
1334	*
1335	* @returns VBox status code.
1336	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1337	* enmFlush).
1338	* @param enmFlush Type of flush.
1339	*
1340	* @remarks Caller is responsible for making sure this function is called only
1341	* when NestedPaging is supported and providing @a enmFlush that is
1342	* supported by the CPU.
1343	* @remarks Can be called with interrupts disabled.
1344	*/
1345	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1346	{
1347	uint64_t au64Descriptor[2];
1348	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1349	au64Descriptor[0] = 0;
1350	else
1351	{
1352	Assert(pVCpu);
1353	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1354	}
1355	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1356
1357	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1358	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1359	rc));
1360	if ( RT_SUCCESS(rc)
1361	&& pVCpu)
1362	{
1363	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1364	}
1365	}
1366
1367
1368	/**
1369	* Flushes the TLB using VPID.
1370	*
1371	* @returns VBox status code.
1372	* @param pVM Pointer to the VM.
1373	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1374	* enmFlush).
1375	* @param enmFlush Type of flush.
1376	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1377	* on @a enmFlush).
1378	*
1379	* @remarks Can be called with interrupts disabled.
1380	*/
1381	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1382	{
1383	AssertPtr(pVM);
1384	Assert(pVM->hm.s.vmx.fVpid);
1385
1386	uint64_t au64Descriptor[2];
1387	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1388	{
1389	au64Descriptor[0] = 0;
1390	au64Descriptor[1] = 0;
1391	}
1392	else
1393	{
1394	AssertPtr(pVCpu);
1395	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1396	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1397	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1398	au64Descriptor[1] = GCPtr;
1399	}
1400
1401	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1402	AssertMsg(rc == VINF_SUCCESS,
1403	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1404	if ( RT_SUCCESS(rc)
1405	&& pVCpu)
1406	{
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1408	}
1409	}
1410
1411
1412	/**
1413	* Invalidates a guest page by guest virtual address. Only relevant for
1414	* EPT/VPID, otherwise there is nothing really to invalidate.
1415	*
1416	* @returns VBox status code.
1417	* @param pVM Pointer to the VM.
1418	* @param pVCpu Pointer to the VMCPU.
1419	* @param GCVirt Guest virtual address of the page to invalidate.
1420	*/
1421	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1422	{
1423	AssertPtr(pVM);
1424	AssertPtr(pVCpu);
1425	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1426
1427	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1428	if (!fFlushPending)
1429	{
1430	/*
1431	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1432	* See @bugref{6043} and @bugref{6177}.
1433	*
1434	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1435	* function maybe called in a loop with individual addresses.
1436	*/
1437	if (pVM->hm.s.vmx.fVpid)
1438	{
1439	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1440	{
1441	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1442	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1443	}
1444	else
1445	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1446	}
1447	else if (pVM->hm.s.fNestedPaging)
1448	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1449	}
1450
1451	return VINF_SUCCESS;
1452	}
1453
1454
1455	/**
1456	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1457	* otherwise there is nothing really to invalidate.
1458	*
1459	* @returns VBox status code.
1460	* @param pVM Pointer to the VM.
1461	* @param pVCpu Pointer to the VMCPU.
1462	* @param GCPhys Guest physical address of the page to invalidate.
1463	*/
1464	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1465	{
1466	LogFlowFunc(("%RGp\n", GCPhys));
1467
1468	/*
1469	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1470	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1471	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1472	*/
1473	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1474	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1475	return VINF_SUCCESS;
1476	}
1477
1478
1479	/**
1480	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1481	* case where neither EPT nor VPID is supported by the CPU.
1482	*
1483	* @param pVM Pointer to the VM.
1484	* @param pVCpu Pointer to the VMCPU.
1485	* @param pCpu Pointer to the global HM struct.
1486	*
1487	* @remarks Called with interrupts disabled.
1488	*/
1489	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1490	{
1491	AssertPtr(pVCpu);
1492	AssertPtr(pCpu);
1493	NOREF(pVM);
1494
1495	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1496
1497	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1498	#if 0
1499	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1500	pVCpu->hm.s.TlbShootdown.cPages = 0;
1501	#endif
1502
1503	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1504	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1505	pVCpu->hm.s.fForceTLBFlush = false;
1506	return;
1507	}
1508
1509
1510	/**
1511	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1512	*
1513	* @param pVM Pointer to the VM.
1514	* @param pVCpu Pointer to the VMCPU.
1515	* @param pCpu Pointer to the global HM CPU struct.
1516	* @remarks All references to "ASID" in this function pertains to "VPID" in
1517	* Intel's nomenclature. The reason is, to avoid confusion in compare
1518	* statements since the host-CPU copies are named "ASID".
1519	*
1520	* @remarks Called with interrupts disabled.
1521	*/
1522	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1523	{
1524	#ifdef VBOX_WITH_STATISTICS
1525	bool fTlbFlushed = false;
1526	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1527	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1528	if (!fTlbFlushed) \
1529	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1530	} while (0)
1531	#else
1532	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1533	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1534	#endif
1535
1536	AssertPtr(pVM);
1537	AssertPtr(pCpu);
1538	AssertPtr(pVCpu);
1539	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1540	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1541	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1542
1543
1544	/*
1545	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1546	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1547	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1548	*/
1549	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1550	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1551	{
1552	++pCpu->uCurrentAsid;
1553	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1554	{
1555	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1556	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1557	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1558	}
1559
1560	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1561	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1562	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1563
1564	/*
1565	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1566	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1567	*/
1568	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1569	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1570	HMVMX_SET_TAGGED_TLB_FLUSHED();
1571	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1572	}
1573
1574	/* Check for explicit TLB shootdowns. */
1575	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1576	{
1577	/*
1578	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1579	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1580	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1581	* but not guest-physical mappings.
1582	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1583	*/
1584	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1585	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1586	HMVMX_SET_TAGGED_TLB_FLUSHED();
1587	}
1588
1589	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1590	* where it is commented out. Support individual entry flushing
1591	* someday. */
1592	#if 0
1593	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1594	{
1595	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1596
1597	/*
1598	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1599	* as supported by the CPU.
1600	*/
1601	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1602	{
1603	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1604	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1605	}
1606	else
1607	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1608
1609	HMVMX_SET_TAGGED_TLB_FLUSHED();
1610	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1611	pVCpu->hm.s.TlbShootdown.cPages = 0;
1612	}
1613	#endif
1614
1615	pVCpu->hm.s.fForceTLBFlush = false;
1616
1617	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1618
1619	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1620	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1621	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1622	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1623	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1624	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1625	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1626	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1627
1628	/* Update VMCS with the VPID. */
1629	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1630	AssertRC(rc);
1631
1632	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1633	}
1634
1635
1636	/**
1637	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1638	*
1639	* @returns VBox status code.
1640	* @param pVM Pointer to the VM.
1641	* @param pVCpu Pointer to the VMCPU.
1642	* @param pCpu Pointer to the global HM CPU struct.
1643	*
1644	* @remarks Called with interrupts disabled.
1645	*/
1646	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1647	{
1648	AssertPtr(pVM);
1649	AssertPtr(pVCpu);
1650	AssertPtr(pCpu);
1651	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1652	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1653
1654	/*
1655	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1656	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1657	*/
1658	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1659	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1660	{
1661	pVCpu->hm.s.fForceTLBFlush = true;
1662	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1663	}
1664
1665	/* Check for explicit TLB shootdown flushes. */
1666	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1667	{
1668	pVCpu->hm.s.fForceTLBFlush = true;
1669	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1670	}
1671
1672	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1673	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1674
1675	if (pVCpu->hm.s.fForceTLBFlush)
1676	{
1677	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1678	pVCpu->hm.s.fForceTLBFlush = false;
1679	}
1680	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1681	* where it is commented out. Support individual entry flushing
1682	* someday. */
1683	#if 0
1684	else
1685	{
1686	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1687	{
1688	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1689	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1690	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1691	}
1692	else
1693	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1694
1695	pVCpu->hm.s.TlbShootdown.cPages = 0;
1696	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1697	}
1698	#endif
1699	}
1700
1701
1702	/**
1703	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1704	*
1705	* @returns VBox status code.
1706	* @param pVM Pointer to the VM.
1707	* @param pVCpu Pointer to the VMCPU.
1708	* @param pCpu Pointer to the global HM CPU struct.
1709	*
1710	* @remarks Called with interrupts disabled.
1711	*/
1712	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1713	{
1714	AssertPtr(pVM);
1715	AssertPtr(pVCpu);
1716	AssertPtr(pCpu);
1717	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1718	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1719
1720	/*
1721	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1722	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1723	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1724	*/
1725	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1726	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1727	{
1728	pVCpu->hm.s.fForceTLBFlush = true;
1729	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1730	}
1731
1732	/* Check for explicit TLB shootdown flushes. */
1733	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1734	{
1735	/*
1736	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1737	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1738	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1739	*/
1740	pVCpu->hm.s.fForceTLBFlush = true;
1741	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1742	}
1743
1744	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1745	if (pVCpu->hm.s.fForceTLBFlush)
1746	{
1747	++pCpu->uCurrentAsid;
1748	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1749	{
1750	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1751	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1752	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1753	}
1754
1755	pVCpu->hm.s.fForceTLBFlush = false;
1756	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1757	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1758	if (pCpu->fFlushAsidBeforeUse)
1759	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1760	}
1761	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1762	* where it is commented out. Support individual entry flushing
1763	* someday. */
1764	#if 0
1765	else
1766	{
1767	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1768	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1769	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1770	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1771
1772	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1773	{
1774	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1775	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1776	{
1777	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1778	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1779	}
1780	else
1781	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1782
1783	pVCpu->hm.s.TlbShootdown.cPages = 0;
1784	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1785	}
1786	else
1787	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1788	}
1789	#endif
1790
1791	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1792	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1793	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1794	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1795	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1796	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1797
1798	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1799	AssertRC(rc);
1800	}
1801
1802
1803	/**
1804	* Flushes the guest TLB entry based on CPU capabilities.
1805	*
1806	* @param pVCpu Pointer to the VMCPU.
1807	* @param pCpu Pointer to the global HM CPU struct.
1808	*/
1809	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1810	{
1811	#ifdef HMVMX_ALWAYS_FLUSH_TLB
1812	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1813	#endif
1814	PVM pVM = pVCpu->CTX_SUFF(pVM);
1815	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1816	{
1817	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1818	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1819	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1820	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1821	default:
1822	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1823	break;
1824	}
1825
1826	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
1827	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
1828
1829	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
1830	}
1831
1832
1833	/**
1834	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1835	* TLB entries from the host TLB before VM-entry.
1836	*
1837	* @returns VBox status code.
1838	* @param pVM Pointer to the VM.
1839	*/
1840	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1841	{
1842	/*
1843	* Determine optimal flush type for Nested Paging.
1844	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1845	* guest execution (see hmR3InitFinalizeR0()).
1846	*/
1847	if (pVM->hm.s.fNestedPaging)
1848	{
1849	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1850	{
1851	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1852	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1853	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1854	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1855	else
1856	{
1857	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1858	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1859	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1860	}
1861
1862	/* Make sure the write-back cacheable memory type for EPT is supported. */
1863	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1864	{
1865	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1866	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1867	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1868	}
1869	}
1870	else
1871	{
1872	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1873	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1874	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1875	}
1876	}
1877
1878	/*
1879	* Determine optimal flush type for VPID.
1880	*/
1881	if (pVM->hm.s.vmx.fVpid)
1882	{
1883	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1884	{
1885	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1886	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1887	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1888	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1889	else
1890	{
1891	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1892	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1893	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1894	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1895	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1896	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1897	pVM->hm.s.vmx.fVpid = false;
1898	}
1899	}
1900	else
1901	{
1902	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1903	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1904	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1905	pVM->hm.s.vmx.fVpid = false;
1906	}
1907	}
1908
1909	/*
1910	* Setup the handler for flushing tagged-TLBs.
1911	*/
1912	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1913	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1914	else if (pVM->hm.s.fNestedPaging)
1915	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1916	else if (pVM->hm.s.vmx.fVpid)
1917	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1918	else
1919	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1920	return VINF_SUCCESS;
1921	}
1922
1923
1924	/**
1925	* Sets up pin-based VM-execution controls in the VMCS.
1926	*
1927	* @returns VBox status code.
1928	* @param pVM Pointer to the VM.
1929	* @param pVCpu Pointer to the VMCPU.
1930	*/
1931	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1932	{
1933	AssertPtr(pVM);
1934	AssertPtr(pVCpu);
1935
1936	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1937	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1938
1939	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1940	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1941	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1942
1943	/* Enable the VMX preemption timer. */
1944	if (pVM->hm.s.vmx.fUsePreemptTimer)
1945	{
1946	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1947	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1948	}
1949
1950	if ((val & zap) != val)
1951	{
1952	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1953	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1954	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1955	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1956	}
1957
1958	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1959	AssertRCReturn(rc, rc);
1960
1961	/* Update VCPU with the currently set pin-based VM-execution controls. */
1962	pVCpu->hm.s.vmx.u32PinCtls = val;
1963	return rc;
1964	}
1965
1966
1967	/**
1968	* Sets up processor-based VM-execution controls in the VMCS.
1969	*
1970	* @returns VBox status code.
1971	* @param pVM Pointer to the VM.
1972	* @param pVMCPU Pointer to the VMCPU.
1973	*/
1974	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1975	{
1976	AssertPtr(pVM);
1977	AssertPtr(pVCpu);
1978
1979	int rc = VERR_INTERNAL_ERROR_5;
1980	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1981	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1982
1983	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1984	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1985	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1986	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1987	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1988	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1989	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1990
1991	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1992	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1993	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1994	{
1995	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1996	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1997	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1998	}
1999
2000	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2001	if (!pVM->hm.s.fNestedPaging)
2002	{
2003	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2004	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2005	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2006	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2007	}
2008
2009	/* Use TPR shadowing if supported by the CPU. */
2010	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2011	{
2012	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2013	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2014	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2015	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2016	AssertRCReturn(rc, rc);
2017
2018	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2019	/* CR8 writes causes a VM-exit based on TPR threshold. */
2020	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2021	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2022	}
2023	else
2024	{
2025	/*
2026	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2027	* Set this control only for 64-bit guests.
2028	*/
2029	if (pVM->hm.s.fAllow64BitGuests)
2030	{
2031	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2032	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2033	}
2034	}
2035
2036	/* Use MSR-bitmaps if supported by the CPU. */
2037	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2038	{
2039	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2040
2041	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2042	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2043	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2044	AssertRCReturn(rc, rc);
2045
2046	/*
2047	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2048	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
2049	*/
2050	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2051	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2052	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2053	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2054	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2055	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2056	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2057	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2058	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2059	}
2060
2061	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2062	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2063	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2064
2065	if ((val & zap) != val)
2066	{
2067	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2068	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2069	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2070	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2071	}
2072
2073	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2074	AssertRCReturn(rc, rc);
2075
2076	/* Update VCPU with the currently set processor-based VM-execution controls. */
2077	pVCpu->hm.s.vmx.u32ProcCtls = val;
2078
2079	/*
2080	* Secondary processor-based VM-execution controls.
2081	*/
2082	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2083	{
2084	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2085	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2086
2087	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2088	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2089
2090	if (pVM->hm.s.fNestedPaging)
2091	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2092	else
2093	{
2094	/*
2095	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2096	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2097	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2098	*/
2099	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2100	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2101	}
2102
2103	if (pVM->hm.s.vmx.fVpid)
2104	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2105
2106	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2107	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2108
2109	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2110	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2111	* done dynamically. */
2112	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2113	{
2114	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2115	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2116	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2117	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2118	AssertRCReturn(rc, rc);
2119	}
2120
2121	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2122	{
2123	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2124	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2125	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2126	}
2127
2128	if ((val & zap) != val)
2129	{
2130	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2131	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2132	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2133	}
2134
2135	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2136	AssertRCReturn(rc, rc);
2137
2138	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2139	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2140	}
2141	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2142	{
2143	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2144	"available\n"));
2145	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2146	}
2147
2148	return VINF_SUCCESS;
2149	}
2150
2151
2152	/**
2153	* Sets up miscellaneous (everything other than Pin & Processor-based
2154	* VM-execution) control fields in the VMCS.
2155	*
2156	* @returns VBox status code.
2157	* @param pVM Pointer to the VM.
2158	* @param pVCpu Pointer to the VMCPU.
2159	*/
2160	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2161	{
2162	AssertPtr(pVM);
2163	AssertPtr(pVCpu);
2164
2165	int rc = VERR_GENERAL_FAILURE;
2166
2167	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2168	#if 0
2169	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2170	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2171	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2172
2173	/*
2174	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2175	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2176	* We thus use the exception bitmap to control it rather than use both.
2177	*/
2178	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2179	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2180
2181	/** @todo Explore possibility of using IO-bitmaps. */
2182	/* All IO & IOIO instructions cause VM-exits. */
2183	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2184	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2185
2186	/* Initialize the MSR-bitmap area. */
2187	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2188	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2189	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2190	#endif
2191
2192	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2193	/* Setup MSR autoloading/storing. */
2194	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2195	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2196	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2197	AssertRCReturn(rc, rc);
2198	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2199	AssertRCReturn(rc, rc);
2200
2201	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2202	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2203	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2204	AssertRCReturn(rc, rc);
2205	#endif
2206
2207	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2208	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2209	AssertRCReturn(rc, rc);
2210
2211	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2212	#if 0
2213	/* Setup debug controls */
2214	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2215	AssertRCReturn(rc, rc);
2216	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2217	AssertRCReturn(rc, rc);
2218	#endif
2219
2220	return rc;
2221	}
2222
2223
2224	/**
2225	* Sets up the initial exception bitmap in the VMCS based on static conditions
2226	* (i.e. conditions that cannot ever change after starting the VM).
2227	*
2228	* @returns VBox status code.
2229	* @param pVM Pointer to the VM.
2230	* @param pVCpu Pointer to the VMCPU.
2231	*/
2232	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2233	{
2234	AssertPtr(pVM);
2235	AssertPtr(pVCpu);
2236
2237	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2238
2239	uint32_t u32XcptBitmap = 0;
2240
2241	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2242	if (!pVM->hm.s.fNestedPaging)
2243	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2244
2245	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2246	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2247	AssertRCReturn(rc, rc);
2248	return rc;
2249	}
2250
2251
2252	/**
2253	* Sets up the initial guest-state mask. The guest-state mask is consulted
2254	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2255	* for the nested virtualization case (as it would cause a VM-exit).
2256	*
2257	* @param pVCpu Pointer to the VMCPU.
2258	*/
2259	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2260	{
2261	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2262	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2263	return VINF_SUCCESS;
2264	}
2265
2266
2267	/**
2268	* Does per-VM VT-x initialization.
2269	*
2270	* @returns VBox status code.
2271	* @param pVM Pointer to the VM.
2272	*/
2273	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2274	{
2275	LogFlowFunc(("pVM=%p\n", pVM));
2276
2277	int rc = hmR0VmxStructsAlloc(pVM);
2278	if (RT_FAILURE(rc))
2279	{
2280	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2281	return rc;
2282	}
2283
2284	return VINF_SUCCESS;
2285	}
2286
2287
2288	/**
2289	* Does per-VM VT-x termination.
2290	*
2291	* @returns VBox status code.
2292	* @param pVM Pointer to the VM.
2293	*/
2294	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2295	{
2296	LogFlowFunc(("pVM=%p\n", pVM));
2297
2298	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2299	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2300	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2301	#endif
2302	hmR0VmxStructsFree(pVM);
2303	return VINF_SUCCESS;
2304	}
2305
2306
2307	/**
2308	* Sets up the VM for execution under VT-x.
2309	* This function is only called once per-VM during initialization.
2310	*
2311	* @returns VBox status code.
2312	* @param pVM Pointer to the VM.
2313	*/
2314	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2315	{
2316	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2317	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2318
2319	LogFlowFunc(("pVM=%p\n", pVM));
2320
2321	/*
2322	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2323	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2324	*/
2325	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2326	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2327	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2328	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2329	{
2330	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2331	return VERR_INTERNAL_ERROR;
2332	}
2333
2334	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335	/*
2336	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2337	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2338	*/
2339	if ( pVM->hm.s.fAllow64BitGuests
2340	&& !HMVMX_IS_64BIT_HOST_MODE())
2341	{
2342	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2343	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2344	}
2345	#endif
2346
2347	/* Initialize these always, see hmR3InitFinalizeR0().*/
2348	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2349	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2350
2351	/* Setup the tagged-TLB flush handlers. */
2352	int rc = hmR0VmxSetupTaggedTlb(pVM);
2353	if (RT_FAILURE(rc))
2354	{
2355	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2356	return rc;
2357	}
2358
2359	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2360	{
2361	PVMCPU pVCpu = &pVM->aCpus[i];
2362	AssertPtr(pVCpu);
2363	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2364
2365	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2366	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2367
2368	/* Set revision dword at the beginning of the VMCS structure. */
2369	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2370
2371	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2372	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2373	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2374	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2375
2376	/* Load this VMCS as the current VMCS. */
2377	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2378	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2379	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2380
2381	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2382	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2383	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2384
2385	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2386	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2387	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2388
2389	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2390	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2391	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2392
2393	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2394	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2395	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2396
2397	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2398	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2399	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2400
2401	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2402	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2403	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2404	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2405	#endif
2406
2407	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2408	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2409	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2410	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2411
2412	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2413
2414	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2415	}
2416
2417	return VINF_SUCCESS;
2418	}
2419
2420
2421	/**
2422	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2423	* the VMCS.
2424	*
2425	* @returns VBox status code.
2426	* @param pVM Pointer to the VM.
2427	* @param pVCpu Pointer to the VMCPU.
2428	*/
2429	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2430	{
2431	RTCCUINTREG uReg = ASMGetCR0();
2432	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2433	AssertRCReturn(rc, rc);
2434
2435	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2436	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2437	if (HMVMX_IS_64BIT_HOST_MODE())
2438	{
2439	uint64_t uRegCR3 = HMR0Get64bitCR3();
2440	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2441	}
2442	else
2443	#endif
2444	{
2445	uReg = ASMGetCR3();
2446	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2447	}
2448	AssertRCReturn(rc, rc);
2449
2450	uReg = ASMGetCR4();
2451	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2452	AssertRCReturn(rc, rc);
2453	return rc;
2454	}
2455
2456
2457	#if HC_ARCH_BITS == 64
2458	/**
2459	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2460	* requirements. See hmR0VmxSaveHostSegmentRegs().
2461	*/
2462	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2463	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2464	{ \
2465	bool fValidSelector = true; \
2466	if ((selValue) & X86_SEL_LDT) \
2467	{ \
2468	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2469	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2470	} \
2471	if (fValidSelector) \
2472	{ \
2473	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2474	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2475	} \
2476	(selValue) = 0; \
2477	}
2478	#endif
2479
2480
2481	/**
2482	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2483	* the host-state area in the VMCS.
2484	*
2485	* @returns VBox status code.
2486	* @param pVM Pointer to the VM.
2487	* @param pVCpu Pointer to the VMCPU.
2488	*/
2489	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2490	{
2491	int rc = VERR_INTERNAL_ERROR_5;
2492
2493	/*
2494	* Host DS, ES, FS and GS segment registers.
2495	*/
2496	#if HC_ARCH_BITS == 64
2497	RTSEL uSelDS = ASMGetDS();
2498	RTSEL uSelES = ASMGetES();
2499	RTSEL uSelFS = ASMGetFS();
2500	RTSEL uSelGS = ASMGetGS();
2501	#else
2502	RTSEL uSelDS = 0;
2503	RTSEL uSelES = 0;
2504	RTSEL uSelFS = 0;
2505	RTSEL uSelGS = 0;
2506	#endif
2507
2508	/* Recalculate which host-state bits need to be manually restored. */
2509	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2510
2511	/*
2512	* Host CS and SS segment registers.
2513	*/
2514	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2515	RTSEL uSelCS;
2516	RTSEL uSelSS;
2517	if (HMVMX_IS_64BIT_HOST_MODE())
2518	{
2519	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2520	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2521	}
2522	else
2523	{
2524	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2525	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2526	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2527	}
2528	#else
2529	RTSEL uSelCS = ASMGetCS();
2530	RTSEL uSelSS = ASMGetSS();
2531	#endif
2532
2533	/*
2534	* Host TR segment register.
2535	*/
2536	RTSEL uSelTR = ASMGetTR();
2537
2538	#if HC_ARCH_BITS == 64
2539	/*
2540	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2541	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2542	*/
2543	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2544	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2545	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2546	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2547	# undef VMXLOCAL_ADJUST_HOST_SEG
2548	#endif
2549
2550	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2551	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2552	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2553	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2554	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2555	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2556	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2557	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2558	Assert(uSelCS);
2559	Assert(uSelTR);
2560
2561	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2562	#if 0
2563	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2564	Assert(uSelSS != 0);
2565	#endif
2566
2567	/* Write these host selector fields into the host-state area in the VMCS. */
2568	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2569	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2570	#if HC_ARCH_BITS == 64
2571	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2572	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2573	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2574	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2575	#endif
2576	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2577
2578	/*
2579	* Host GDTR and IDTR.
2580	*/
2581	RTGDTR Gdtr;
2582	RT_ZERO(Gdtr);
2583	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2584	if (HMVMX_IS_64BIT_HOST_MODE())
2585	{
2586	X86XDTR64 Gdtr64;
2587	X86XDTR64 Idtr64;
2588	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2589	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2590	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2591
2592	Gdtr.cbGdt = Gdtr64.cb;
2593	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2594	}
2595	else
2596	#endif
2597	{
2598	RTIDTR Idtr;
2599	ASMGetGDTR(&Gdtr);
2600	ASMGetIDTR(&Idtr);
2601	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2602	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2603
2604	#if HC_ARCH_BITS == 64
2605	/*
2606	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2607	* maximum limit (0xffff) on every VM-exit.
2608	*/
2609	if (Gdtr.cbGdt != 0xffff)
2610	{
2611	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2612	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2613	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2614	}
2615
2616	/*
2617	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2618	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2619	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2620	*/
2621	if (Idtr.cbIdt < 0x0fff)
2622	{
2623	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2624	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2625	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2626	}
2627	#endif
2628	}
2629
2630	/*
2631	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2632	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2633	*/
2634	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2635	{
2636	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2637	return VERR_VMX_INVALID_HOST_STATE;
2638	}
2639
2640	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2641	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2642	if (HMVMX_IS_64BIT_HOST_MODE())
2643	{
2644	/* We need the 64-bit TR base for hybrid darwin. */
2645	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2646	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2647	}
2648	else
2649	#endif
2650	{
2651	uintptr_t uTRBase;
2652	#if HC_ARCH_BITS == 64
2653	uTRBase = X86DESC64_BASE(pDesc);
2654
2655	/*
2656	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2657	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2658	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2659	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2660	*
2661	* [1] See Intel spec. 3.5 "System Descriptor Types".
2662	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2663	*/
2664	Assert(pDesc->System.u4Type == 11);
2665	if ( pDesc->System.u16LimitLow != 0x67
2666	\|\| pDesc->System.u4LimitHigh)
2667	{
2668	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2669	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2670
2671	/* Store the GDTR here as we need it while restoring TR. */
2672	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2673	}
2674	#else
2675	uTRBase = X86DESC_BASE(pDesc);
2676	#endif
2677	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2678	}
2679	AssertRCReturn(rc, rc);
2680
2681	/*
2682	* Host FS base and GS base.
2683	*/
2684	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2685	if (HMVMX_IS_64BIT_HOST_MODE())
2686	{
2687	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2688	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2689	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2690	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2691
2692	# if HC_ARCH_BITS == 64
2693	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2694	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2695	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2696	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2697	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2698	# endif
2699	}
2700	#endif
2701	return rc;
2702	}
2703
2704
2705	/**
2706	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2707	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2708	* the host after every successful VM exit.
2709	*
2710	* @returns VBox status code.
2711	* @param pVM Pointer to the VM.
2712	* @param pVCpu Pointer to the VMCPU.
2713	*/
2714	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2715	{
2716	AssertPtr(pVCpu);
2717	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2718
2719	int rc = VINF_SUCCESS;
2720	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2721	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2722	uint32_t cHostMsrs = 0;
2723	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2724
2725	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2726	{
2727	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2728
2729	# if HC_ARCH_BITS == 64
2730	/* Paranoia. 64-bit code requires these bits to be set always. */
2731	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2732
2733	/*
2734	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2735	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2736	* some reason (e.g. allow transparent reads) we would activate the code below.
2737	*/
2738	# if 0
2739	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2740	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2741	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2742	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2743	if (CPUMIsGuestInLongMode(pVCpu))
2744	{
2745	uint64_t u64GuestEfer;
2746	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2747	AssertRC(rc);
2748
2749	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2750	{
2751	pHostMsr->u32Msr = MSR_K6_EFER;
2752	pHostMsr->u32Reserved = 0;
2753	pHostMsr->u64Value = u64HostEfer;
2754	pHostMsr++; cHostMsrs++;
2755	}
2756	}
2757	# endif
2758	# else /* HC_ARCH_BITS != 64 */
2759	pHostMsr->u32Msr = MSR_K6_EFER;
2760	pHostMsr->u32Reserved = 0;
2761	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2762	if (CPUMIsGuestInLongMode(pVCpu))
2763	{
2764	/* Must match the EFER value in our 64 bits switcher. */
2765	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2766	}
2767	else
2768	# endif
2769	pHostMsr->u64Value = u64HostEfer;
2770	pHostMsr++; cHostMsrs++;
2771	# endif /* HC_ARCH_BITS == 64 */
2772	}
2773
2774	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2775	if (HMVMX_IS_64BIT_HOST_MODE())
2776	{
2777	pHostMsr->u32Msr = MSR_K6_STAR;
2778	pHostMsr->u32Reserved = 0;
2779	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2780	pHostMsr++; cHostMsrs++;
2781	pHostMsr->u32Msr = MSR_K8_LSTAR;
2782	pHostMsr->u32Reserved = 0;
2783	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2784	pHostMsr++; cHostMsrs++;
2785	pHostMsr->u32Msr = MSR_K8_SF_MASK;
2786	pHostMsr->u32Reserved = 0;
2787	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2788	pHostMsr++; cHostMsrs++;
2789	pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2790	pHostMsr->u32Reserved = 0;
2791	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2792	pHostMsr++; cHostMsrs++;
2793	}
2794	# endif
2795
2796	/* Host TSC AUX MSR must be restored since we always load/store guest TSC AUX MSR. */
2797	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2798	{
2799	pHostMsr->u32Msr = MSR_K8_TSC_AUX;
2800	pHostMsr->u32Reserved = 0;
2801	pHostMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
2802	pHostMsr++; cHostMsrs++;
2803	}
2804
2805	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2806	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2807	{
2808	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2809	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2810	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2811	}
2812
2813	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2814	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2815
2816	/*
2817	* Host Sysenter MSRs.
2818	*/
2819	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2820	AssertRCReturn(rc, rc);
2821	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2822	if (HMVMX_IS_64BIT_HOST_MODE())
2823	{
2824	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2825	AssertRCReturn(rc, rc);
2826	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2827	}
2828	else
2829	{
2830	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2831	AssertRCReturn(rc, rc);
2832	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2833	}
2834	#elif HC_ARCH_BITS == 32
2835	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2836	AssertRCReturn(rc, rc);
2837	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2838	#else
2839	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2840	AssertRCReturn(rc, rc);
2841	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2842	#endif
2843	AssertRCReturn(rc, rc);
2844
2845	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2846	* hmR0VmxSetupExitCtls() !! */
2847	return rc;
2848	}
2849
2850
2851	/**
2852	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2853	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2854	* controls".
2855	*
2856	* @returns VBox status code.
2857	* @param pVCpu Pointer to the VMCPU.
2858	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2859	* out-of-sync. Make sure to update the required fields
2860	* before using them.
2861	*
2862	* @remarks No-long-jump zone!!!
2863	*/
2864	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2865	{
2866	int rc = VINF_SUCCESS;
2867	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2868	{
2869	PVM pVM = pVCpu->CTX_SUFF(pVM);
2870	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2871	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2872
2873	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2874	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2875
2876	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2877	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2878	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2879	else
2880	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2881
2882	/*
2883	* The following should -not- be set (since we're not in SMM mode):
2884	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2885	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2886	*/
2887
2888	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2889	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2890	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2891
2892	if ((val & zap) != val)
2893	{
2894	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2895	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2896	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2897	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2898	}
2899
2900	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2901	AssertRCReturn(rc, rc);
2902
2903	/* Update VCPU with the currently set VM-exit controls. */
2904	pVCpu->hm.s.vmx.u32EntryCtls = val;
2905	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2906	}
2907	return rc;
2908	}
2909
2910
2911	/**
2912	* Sets up the VM-exit controls in the VMCS.
2913	*
2914	* @returns VBox status code.
2915	* @param pVM Pointer to the VM.
2916	* @param pVCpu Pointer to the VMCPU.
2917	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2918	* out-of-sync. Make sure to update the required fields
2919	* before using them.
2920	*
2921	* @remarks requires EFER.
2922	*/
2923	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2924	{
2925	int rc = VINF_SUCCESS;
2926	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2927	{
2928	PVM pVM = pVCpu->CTX_SUFF(pVM);
2929	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2930	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2931
2932	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2933	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2934
2935	/*
2936	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2937	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2938	*/
2939	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2940	if (HMVMX_IS_64BIT_HOST_MODE())
2941	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2942	else
2943	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2944	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2945	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2946	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2947	else
2948	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2949	#endif
2950
2951	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2952	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2953
2954	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2955	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2956	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2957	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2958	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2959
2960	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2961	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2962
2963	if ((val & zap) != val)
2964	{
2965	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2966	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2967	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2968	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2969	}
2970
2971	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2972	AssertRCReturn(rc, rc);
2973
2974	/* Update VCPU with the currently set VM-exit controls. */
2975	pVCpu->hm.s.vmx.u32ExitCtls = val;
2976	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2977	}
2978	return rc;
2979	}
2980
2981
2982	/**
2983	* Loads the guest APIC and related state.
2984	*
2985	* @returns VBox status code.
2986	* @param pVM Pointer to the VM.
2987	* @param pVCpu Pointer to the VMCPU.
2988	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2989	* out-of-sync. Make sure to update the required fields
2990	* before using them.
2991	*/
2992	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2993	{
2994	int rc = VINF_SUCCESS;
2995	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2996	{
2997	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2998	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2999	{
3000	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3001
3002	bool fPendingIntr = false;
3003	uint8_t u8Tpr = 0;
3004	uint8_t u8PendingIntr = 0;
3005	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3006	AssertRCReturn(rc, rc);
3007
3008	/*
3009	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3010	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3011	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3012	* the interrupt when we VM-exit for other reasons.
3013	*/
3014	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3015	uint32_t u32TprThreshold = 0;
3016	if (fPendingIntr)
3017	{
3018	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3019	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3020	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3021	if (u8PendingPriority <= u8TprPriority)
3022	u32TprThreshold = u8PendingPriority;
3023	else
3024	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3025	}
3026	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3027
3028	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3029	AssertRCReturn(rc, rc);
3030	}
3031
3032	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3033	}
3034	return rc;
3035	}
3036
3037
3038	/**
3039	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3040	*
3041	* @returns Guest's interruptibility-state.
3042	* @param pVCpu Pointer to the VMCPU.
3043	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3044	* out-of-sync. Make sure to update the required fields
3045	* before using them.
3046	*
3047	* @remarks No-long-jump zone!!!
3048	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3049	*/
3050	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3051	{
3052	/*
3053	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3054	* inhibit interrupts or clear any existing interrupt-inhibition.
3055	*/
3056	uint32_t uIntrState = 0;
3057	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3058	{
3059	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3060	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
3061	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
3062	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3063	{
3064	/*
3065	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3066	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3067	*/
3068	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3069	}
3070	else if (pMixedCtx->eflags.Bits.u1IF)
3071	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3072	else
3073	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3074	}
3075	return uIntrState;
3076	}
3077
3078
3079	/**
3080	* Loads the guest's interruptibility-state into the guest-state area in the
3081	* VMCS.
3082	*
3083	* @returns VBox status code.
3084	* @param pVCpu Pointer to the VMCPU.
3085	* @param uIntrState The interruptibility-state to set.
3086	*/
3087	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3088	{
3089	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3090	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3091	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3092	AssertRCReturn(rc, rc);
3093	return rc;
3094	}
3095
3096
3097	/**
3098	* Loads the guest's RIP into the guest-state area in the VMCS.
3099	*
3100	* @returns VBox status code.
3101	* @param pVCpu Pointer to the VMCPU.
3102	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3103	* out-of-sync. Make sure to update the required fields
3104	* before using them.
3105	*
3106	* @remarks No-long-jump zone!!!
3107	*/
3108	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3109	{
3110	int rc = VINF_SUCCESS;
3111	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3112	{
3113	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3114	AssertRCReturn(rc, rc);
3115
3116	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3117	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
3118	}
3119	return rc;
3120	}
3121
3122
3123	/**
3124	* Loads the guest's RSP into the guest-state area in the VMCS.
3125	*
3126	* @returns VBox status code.
3127	* @param pVCpu Pointer to the VMCPU.
3128	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3129	* out-of-sync. Make sure to update the required fields
3130	* before using them.
3131	*
3132	* @remarks No-long-jump zone!!!
3133	*/
3134	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3135	{
3136	int rc = VINF_SUCCESS;
3137	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3138	{
3139	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3140	AssertRCReturn(rc, rc);
3141
3142	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3143	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3144	}
3145	return rc;
3146	}
3147
3148
3149	/**
3150	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3151	*
3152	* @returns VBox status code.
3153	* @param pVCpu Pointer to the VMCPU.
3154	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3155	* out-of-sync. Make sure to update the required fields
3156	* before using them.
3157	*
3158	* @remarks No-long-jump zone!!!
3159	*/
3160	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3161	{
3162	int rc = VINF_SUCCESS;
3163	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3164	{
3165	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3166	Let us assert it as such and use 32-bit VMWRITE. */
3167	Assert(!(pMixedCtx->rflags.u64 >> 32));
3168	X86EFLAGS Eflags = pMixedCtx->eflags;
3169	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3170	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3171
3172	/*
3173	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
3174	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3175	*/
3176	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3177	{
3178	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3179	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3180	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3181	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3182	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3183	}
3184
3185	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3186	AssertRCReturn(rc, rc);
3187
3188	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3189	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3190	}
3191	return rc;
3192	}
3193
3194
3195	/**
3196	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3197	*
3198	* @returns VBox status code.
3199	* @param pVCpu Pointer to the VMCPU.
3200	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3201	* out-of-sync. Make sure to update the required fields
3202	* before using them.
3203	*
3204	* @remarks No-long-jump zone!!!
3205	*/
3206	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3207	{
3208	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3209	AssertRCReturn(rc, rc);
3210	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3211	AssertRCReturn(rc, rc);
3212	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3213	AssertRCReturn(rc, rc);
3214	return rc;
3215	}
3216
3217
3218	/**
3219	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3220	* CR0 is partially shared with the host and we have to consider the FPU bits.
3221	*
3222	* @returns VBox status code.
3223	* @param pVM Pointer to the VM.
3224	* @param pVCpu Pointer to the VMCPU.
3225	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3226	* out-of-sync. Make sure to update the required fields
3227	* before using them.
3228	*
3229	* @remarks No-long-jump zone!!!
3230	*/
3231	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232	{
3233	/*
3234	* Guest CR0.
3235	* Guest FPU.
3236	*/
3237	int rc = VINF_SUCCESS;
3238	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3239	{
3240	Assert(!(pMixedCtx->cr0 >> 32));
3241	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3242	PVM pVM = pVCpu->CTX_SUFF(pVM);
3243
3244	/* The guest's view (read access) of its CR0 is unblemished. */
3245	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3246	AssertRCReturn(rc, rc);
3247	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3248
3249	/* Setup VT-x's view of the guest CR0. */
3250	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3251	if (pVM->hm.s.fNestedPaging)
3252	{
3253	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3254	{
3255	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3256	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3257	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3258	}
3259	else
3260	{
3261	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3262	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3263	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3264	}
3265
3266	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3267	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3268	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3269
3270	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3271	AssertRCReturn(rc, rc);
3272	}
3273	else
3274	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3275
3276	/*
3277	* Guest FPU bits.
3278	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3279	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3280	*/
3281	u32GuestCR0 \|= X86_CR0_NE;
3282	bool fInterceptNM = false;
3283	if (CPUMIsGuestFPUStateActive(pVCpu))
3284	{
3285	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3286	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3287	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3288	}
3289	else
3290	{
3291	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3292	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3293	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3294	}
3295
3296	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3297	bool fInterceptMF = false;
3298	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3299	fInterceptMF = true;
3300
3301	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3302	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3303	{
3304	Assert(PDMVmmDevHeapIsEnabled(pVM));
3305	Assert(pVM->hm.s.vmx.pRealModeTSS);
3306	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3307	fInterceptNM = true;
3308	fInterceptMF = true;
3309	}
3310	else
3311	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3312
3313	if (fInterceptNM)
3314	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3315	else
3316	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3317
3318	if (fInterceptMF)
3319	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3320	else
3321	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3322
3323	/* Additional intercepts for debugging, define these yourself explicitly. */
3324	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3325	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3326	\| RT_BIT(X86_XCPT_BP)
3327	\| RT_BIT(X86_XCPT_DB)
3328	\| RT_BIT(X86_XCPT_DE)
3329	\| RT_BIT(X86_XCPT_NM)
3330	\| RT_BIT(X86_XCPT_UD)
3331	\| RT_BIT(X86_XCPT_NP)
3332	\| RT_BIT(X86_XCPT_SS)
3333	\| RT_BIT(X86_XCPT_GP)
3334	\| RT_BIT(X86_XCPT_PF)
3335	\| RT_BIT(X86_XCPT_MF)
3336	;
3337	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3338	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3339	#endif
3340
3341	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3342
3343	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3344	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3345	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3346	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3347	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3348	else
3349	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3350
3351	u32GuestCR0 \|= uSetCR0;
3352	u32GuestCR0 &= uZapCR0;
3353	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3354
3355	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3356	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3357	AssertRCReturn(rc, rc);
3358	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3359	AssertRCReturn(rc, rc);
3360	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3361
3362	/*
3363	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3364	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3365	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3366	*/
3367	uint32_t u32CR0Mask = 0;
3368	u32CR0Mask = X86_CR0_PE
3369	\| X86_CR0_NE
3370	\| X86_CR0_WP
3371	\| X86_CR0_PG
3372	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3373	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3374	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3375
3376	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3377	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3378	* and @bugref{6944}. */
3379	#if 0
3380	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3381	u32CR0Mask &= ~X86_CR0_PE;
3382	#endif
3383	if (pVM->hm.s.fNestedPaging)
3384	u32CR0Mask &= ~X86_CR0_WP;
3385
3386	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3387	if (fInterceptNM)
3388	{
3389	u32CR0Mask \|= X86_CR0_TS
3390	\| X86_CR0_MP;
3391	}
3392
3393	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3394	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3395	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3396	AssertRCReturn(rc, rc);
3397	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3398
3399	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3400	}
3401	return rc;
3402	}
3403
3404
3405	/**
3406	* Loads the guest control registers (CR3, CR4) into the guest-state area
3407	* in the VMCS.
3408	*
3409	* @returns VBox status code.
3410	* @param pVM Pointer to the VM.
3411	* @param pVCpu Pointer to the VMCPU.
3412	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3413	* out-of-sync. Make sure to update the required fields
3414	* before using them.
3415	*
3416	* @remarks No-long-jump zone!!!
3417	*/
3418	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3419	{
3420	int rc = VINF_SUCCESS;
3421	PVM pVM = pVCpu->CTX_SUFF(pVM);
3422
3423	/*
3424	* Guest CR2.
3425	* It's always loaded in the assembler code. Nothing to do here.
3426	*/
3427
3428	/*
3429	* Guest CR3.
3430	*/
3431	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3432	{
3433	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3434	if (pVM->hm.s.fNestedPaging)
3435	{
3436	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3437
3438	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3439	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3440	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3441	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3442
3443	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3444	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3445	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3446
3447	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3448	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3449	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3450	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3451
3452	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3453	AssertRCReturn(rc, rc);
3454	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3455
3456	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3457	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3458	{
3459	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3460	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3461	{
3462	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3463	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3464	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3465	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3466	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3467	}
3468
3469	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3470	have Unrestricted Execution to handle the guest when it's not using paging. */
3471	GCPhysGuestCR3 = pMixedCtx->cr3;
3472	}
3473	else
3474	{
3475	/*
3476	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3477	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3478	* EPT takes care of translating it to host-physical addresses.
3479	*/
3480	RTGCPHYS GCPhys;
3481	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3482	Assert(PDMVmmDevHeapIsEnabled(pVM));
3483
3484	/* We obtain it here every time as the guest could have relocated this PCI region. */
3485	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3486	AssertRCReturn(rc, rc);
3487
3488	GCPhysGuestCR3 = GCPhys;
3489	}
3490
3491	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3492	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3493	}
3494	else
3495	{
3496	/* Non-nested paging case, just use the hypervisor's CR3. */
3497	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3498
3499	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3500	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3501	}
3502	AssertRCReturn(rc, rc);
3503
3504	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3505	}
3506
3507	/*
3508	* Guest CR4.
3509	*/
3510	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3511	{
3512	Assert(!(pMixedCtx->cr4 >> 32));
3513	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3514
3515	/* The guest's view of its CR4 is unblemished. */
3516	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3517	AssertRCReturn(rc, rc);
3518	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3519
3520	/* Setup VT-x's view of the guest CR4. */
3521	/*
3522	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3523	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3524	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3525	*/
3526	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3527	{
3528	Assert(pVM->hm.s.vmx.pRealModeTSS);
3529	Assert(PDMVmmDevHeapIsEnabled(pVM));
3530	u32GuestCR4 &= ~X86_CR4_VME;
3531	}
3532
3533	if (pVM->hm.s.fNestedPaging)
3534	{
3535	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3536	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3537	{
3538	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3539	u32GuestCR4 \|= X86_CR4_PSE;
3540	/* Our identity mapping is a 32 bits page directory. */
3541	u32GuestCR4 &= ~X86_CR4_PAE;
3542	}
3543	/* else use guest CR4.*/
3544	}
3545	else
3546	{
3547	/*
3548	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3549	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3550	*/
3551	switch (pVCpu->hm.s.enmShadowMode)
3552	{
3553	case PGMMODE_REAL: /* Real-mode. */
3554	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3555	case PGMMODE_32_BIT: /* 32-bit paging. */
3556	{
3557	u32GuestCR4 &= ~X86_CR4_PAE;
3558	break;
3559	}
3560
3561	case PGMMODE_PAE: /* PAE paging. */
3562	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3563	{
3564	u32GuestCR4 \|= X86_CR4_PAE;
3565	break;
3566	}
3567
3568	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3569	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3570	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3571	break;
3572	#endif
3573	default:
3574	AssertFailed();
3575	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3576	}
3577	}
3578
3579	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3580	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3581	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3582	u32GuestCR4 \|= uSetCR4;
3583	u32GuestCR4 &= uZapCR4;
3584
3585	/* Write VT-x's view of the guest CR4 into the VMCS. */
3586	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3587	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3588	AssertRCReturn(rc, rc);
3589
3590	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3591	uint32_t u32CR4Mask = 0;
3592	u32CR4Mask = X86_CR4_VME
3593	\| X86_CR4_PAE
3594	\| X86_CR4_PGE
3595	\| X86_CR4_PSE
3596	\| X86_CR4_VMXE;
3597	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3598	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3599	AssertRCReturn(rc, rc);
3600
3601	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3602	}
3603	return rc;
3604	}
3605
3606
3607	/**
3608	* Loads the guest debug registers into the guest-state area in the VMCS.
3609	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3610	*
3611	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3612	*
3613	* @returns VBox status code.
3614	* @param pVCpu Pointer to the VMCPU.
3615	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616	* out-of-sync. Make sure to update the required fields
3617	* before using them.
3618	*
3619	* @remarks No-long-jump zone!!!
3620	*/
3621	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622	{
3623	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3624	return VINF_SUCCESS;
3625
3626	#ifdef VBOX_STRICT
3627	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3628	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3629	{
3630	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3631	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3632	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3633	}
3634	#endif
3635
3636	int rc;
3637	PVM pVM = pVCpu->CTX_SUFF(pVM);
3638	bool fInterceptDB = false;
3639	bool fInterceptMovDRx = false;
3640	if (pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu))
3641	{
3642	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3643	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3644	{
3645	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3646	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3647	AssertRCReturn(rc, rc);
3648	Assert(fInterceptDB == false);
3649	}
3650	else
3651	{
3652	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3653	pVCpu->hm.s.fClearTrapFlag = true;
3654	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3655	fInterceptDB = true;
3656	}
3657	}
3658
3659	if (fInterceptDB \|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3660	{
3661	/*
3662	* Use the combined guest and host DRx values found in the hypervisor
3663	* register set because the debugger has breakpoints active or someone
3664	* is single stepping on the host side without a monitor trap flag.
3665	*
3666	* Note! DBGF expects a clean DR6 state before executing guest code.
3667	*/
3668	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3669	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3670	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3671	{
3672	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3673	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3674	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3675	}
3676	else
3677	#endif
3678	if (!CPUMIsHyperDebugStateActive(pVCpu))
3679	{
3680	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3681	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3682	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3683	}
3684
3685	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3686	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3687	AssertRCReturn(rc, rc);
3688
3689	pVCpu->hm.s.fUsingHyperDR7 = true;
3690	fInterceptDB = true;
3691	fInterceptMovDRx = true;
3692	}
3693	else
3694	{
3695	/*
3696	* If the guest has enabled debug registers, we need to load them prior to
3697	* executing guest code so they'll trigger at the right time.
3698	*/
3699	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3700	{
3701	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3702	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3703	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3704	{
3705	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3706	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3707	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3708	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3709	}
3710	else
3711	#endif
3712	if (!CPUMIsGuestDebugStateActive(pVCpu))
3713	{
3714	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3715	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3716	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3717	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3718	}
3719	Assert(!fInterceptDB);
3720	Assert(!fInterceptMovDRx);
3721	}
3722	/*
3723	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3724	* must intercept #DB in order to maintain a correct DR6 guest value.
3725	*/
3726	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3727	else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3728	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3729	\|\| !CPUMIsGuestDebugStateActive(pVCpu))
3730	#else
3731	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3732	#endif
3733	{
3734	fInterceptMovDRx = true;
3735	fInterceptDB = true;
3736	}
3737
3738	/* Update guest DR7. */
3739	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3740	AssertRCReturn(rc, rc);
3741
3742	pVCpu->hm.s.fUsingHyperDR7 = false;
3743	}
3744
3745	/*
3746	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3747	*/
3748	if (fInterceptDB)
3749	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3750	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3751	{
3752	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3753	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3754	#endif
3755	}
3756	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3757	AssertRCReturn(rc, rc);
3758
3759	/*
3760	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3761	*/
3762	if (fInterceptMovDRx)
3763	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3764	else
3765	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3766	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3767	AssertRCReturn(rc, rc);
3768
3769	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3770	return VINF_SUCCESS;
3771	}
3772
3773
3774	#ifdef VBOX_STRICT
3775	/**
3776	* Strict function to validate segment registers.
3777	*
3778	* @remarks ASSUMES CR0 is up to date.
3779	*/
3780	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3781	{
3782	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3783	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3784	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3785	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3786	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3787	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3788	{
3789	/* Protected mode checks */
3790	/* CS */
3791	Assert(pCtx->cs.Attr.n.u1Present);
3792	Assert(!(pCtx->cs.Attr.u & 0xf00));
3793	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3794	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3795	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3796	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3797	\|\| (pCtx->cs.Attr.n.u1Granularity));
3798	/* CS cannot be loaded with NULL in protected mode. */
3799	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3800	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3801	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3802	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3803	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3804	else
3805	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3806	/* SS */
3807	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3808	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3809	if ( !(pCtx->cr0 & X86_CR0_PE)
3810	\|\| pCtx->cs.Attr.n.u4Type == 3)
3811	{
3812	Assert(!pCtx->ss.Attr.n.u2Dpl);
3813	}
3814	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3815	{
3816	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3817	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3818	Assert(pCtx->ss.Attr.n.u1Present);
3819	Assert(!(pCtx->ss.Attr.u & 0xf00));
3820	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3821	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3822	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3823	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3824	\|\| (pCtx->ss.Attr.n.u1Granularity));
3825	}
3826	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3827	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3828	{
3829	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3830	Assert(pCtx->ds.Attr.n.u1Present);
3831	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3832	Assert(!(pCtx->ds.Attr.u & 0xf00));
3833	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3834	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3835	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3836	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3837	\|\| (pCtx->ds.Attr.n.u1Granularity));
3838	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3839	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3840	}
3841	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3842	{
3843	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3844	Assert(pCtx->es.Attr.n.u1Present);
3845	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3846	Assert(!(pCtx->es.Attr.u & 0xf00));
3847	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3848	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3849	\|\| !(pCtx->es.Attr.n.u1Granularity));
3850	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3851	\|\| (pCtx->es.Attr.n.u1Granularity));
3852	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3853	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3854	}
3855	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3856	{
3857	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3858	Assert(pCtx->fs.Attr.n.u1Present);
3859	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3860	Assert(!(pCtx->fs.Attr.u & 0xf00));
3861	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3862	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3863	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3864	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3865	\|\| (pCtx->fs.Attr.n.u1Granularity));
3866	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3867	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3868	}
3869	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3870	{
3871	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3872	Assert(pCtx->gs.Attr.n.u1Present);
3873	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3874	Assert(!(pCtx->gs.Attr.u & 0xf00));
3875	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3876	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3877	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3878	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3879	\|\| (pCtx->gs.Attr.n.u1Granularity));
3880	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3881	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3882	}
3883	/* 64-bit capable CPUs. */
3884	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3885	Assert(!(pCtx->cs.u64Base >> 32));
3886	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3887	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3888	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3889	# endif
3890	}
3891	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3892	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3893	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3894	{
3895	/* Real and v86 mode checks. */
3896	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3897	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3898	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3899	{
3900	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3901	}
3902	else
3903	{
3904	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3905	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3906	}
3907
3908	/* CS */
3909	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3910	Assert(pCtx->cs.u32Limit == 0xffff);
3911	Assert(u32CSAttr == 0xf3);
3912	/* SS */
3913	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3914	Assert(pCtx->ss.u32Limit == 0xffff);
3915	Assert(u32SSAttr == 0xf3);
3916	/* DS */
3917	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3918	Assert(pCtx->ds.u32Limit == 0xffff);
3919	Assert(u32DSAttr == 0xf3);
3920	/* ES */
3921	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3922	Assert(pCtx->es.u32Limit == 0xffff);
3923	Assert(u32ESAttr == 0xf3);
3924	/* FS */
3925	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3926	Assert(pCtx->fs.u32Limit == 0xffff);
3927	Assert(u32FSAttr == 0xf3);
3928	/* GS */
3929	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3930	Assert(pCtx->gs.u32Limit == 0xffff);
3931	Assert(u32GSAttr == 0xf3);
3932	/* 64-bit capable CPUs. */
3933	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	Assert(!(pCtx->cs.u64Base >> 32));
3935	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3936	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3937	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3938	# endif
3939	}
3940	}
3941	#endif /* VBOX_STRICT */
3942
3943
3944	/**
3945	* Writes a guest segment register into the guest-state area in the VMCS.
3946	*
3947	* @returns VBox status code.
3948	* @param pVCpu Pointer to the VMCPU.
3949	* @param idxSel Index of the selector in the VMCS.
3950	* @param idxLimit Index of the segment limit in the VMCS.
3951	* @param idxBase Index of the segment base in the VMCS.
3952	* @param idxAccess Index of the access rights of the segment in the VMCS.
3953	* @param pSelReg Pointer to the segment selector.
3954	* @param pCtx Pointer to the guest-CPU context.
3955	*
3956	* @remarks No-long-jump zone!!!
3957	*/
3958	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3959	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3960	{
3961	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3962	AssertRCReturn(rc, rc);
3963	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3964	AssertRCReturn(rc, rc);
3965	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3966	AssertRCReturn(rc, rc);
3967
3968	uint32_t u32Access = pSelReg->Attr.u;
3969	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3970	{
3971	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3972	u32Access = 0xf3;
3973	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3974	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3975	}
3976	else
3977	{
3978	/*
3979	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3980	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3981	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3982	* loaded in protected-mode have their attribute as 0.
3983	*/
3984	if (!u32Access)
3985	u32Access = X86DESCATTR_UNUSABLE;
3986	}
3987
3988	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3989	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3990	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3991
3992	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3993	AssertRCReturn(rc, rc);
3994	return rc;
3995	}
3996
3997
3998	/**
3999	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4000	* into the guest-state area in the VMCS.
4001	*
4002	* @returns VBox status code.
4003	* @param pVM Pointer to the VM.
4004	* @param pVCPU Pointer to the VMCPU.
4005	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4006	* out-of-sync. Make sure to update the required fields
4007	* before using them.
4008	*
4009	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4010	* @remarks No-long-jump zone!!!
4011	*/
4012	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4013	{
4014	int rc = VERR_INTERNAL_ERROR_5;
4015	PVM pVM = pVCpu->CTX_SUFF(pVM);
4016
4017	/*
4018	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4019	*/
4020	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4021	{
4022	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4023	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4024	{
4025	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4026	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4027	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4028	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4029	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4030	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4031	}
4032
4033	#ifdef VBOX_WITH_REM
4034	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4035	{
4036	Assert(pVM->hm.s.vmx.pRealModeTSS);
4037	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4038	if ( pVCpu->hm.s.vmx.fWasInRealMode
4039	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4040	{
4041	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4042	in real-mode (e.g. OpenBSD 4.0) */
4043	REMFlushTBs(pVM);
4044	Log4(("Load: Switch to protected mode detected!\n"));
4045	pVCpu->hm.s.vmx.fWasInRealMode = false;
4046	}
4047	}
4048	#endif
4049	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4050	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
4051	AssertRCReturn(rc, rc);
4052	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4053	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
4054	AssertRCReturn(rc, rc);
4055	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4056	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
4057	AssertRCReturn(rc, rc);
4058	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4059	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
4060	AssertRCReturn(rc, rc);
4061	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4062	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
4063	AssertRCReturn(rc, rc);
4064	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4065	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
4066	AssertRCReturn(rc, rc);
4067
4068	#ifdef VBOX_STRICT
4069	/* Validate. */
4070	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4071	#endif
4072
4073	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4074	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4075	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4076	}
4077
4078	/*
4079	* Guest TR.
4080	*/
4081	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4082	{
4083	/*
4084	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4085	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4086	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4087	*/
4088	uint16_t u16Sel = 0;
4089	uint32_t u32Limit = 0;
4090	uint64_t u64Base = 0;
4091	uint32_t u32AccessRights = 0;
4092
4093	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4094	{
4095	u16Sel = pMixedCtx->tr.Sel;
4096	u32Limit = pMixedCtx->tr.u32Limit;
4097	u64Base = pMixedCtx->tr.u64Base;
4098	u32AccessRights = pMixedCtx->tr.Attr.u;
4099	}
4100	else
4101	{
4102	Assert(pVM->hm.s.vmx.pRealModeTSS);
4103	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4104
4105	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4106	RTGCPHYS GCPhys;
4107	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4108	AssertRCReturn(rc, rc);
4109
4110	X86DESCATTR DescAttr;
4111	DescAttr.u = 0;
4112	DescAttr.n.u1Present = 1;
4113	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4114
4115	u16Sel = 0;
4116	u32Limit = HM_VTX_TSS_SIZE;
4117	u64Base = GCPhys; /* in real-mode phys = virt. */
4118	u32AccessRights = DescAttr.u;
4119	}
4120
4121	/* Validate. */
4122	Assert(!(u16Sel & RT_BIT(2)));
4123	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4124	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4125	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4126	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4127	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4128	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4129	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4130	Assert( (u32Limit & 0xfff) == 0xfff
4131	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4132	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4133	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4134
4135	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4136	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4137	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4138	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4139
4140	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4141	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4142	}
4143
4144	/*
4145	* Guest GDTR.
4146	*/
4147	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4148	{
4149	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4150	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4151
4152	/* Validate. */
4153	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4154
4155	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4156	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4157	}
4158
4159	/*
4160	* Guest LDTR.
4161	*/
4162	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4163	{
4164	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4165	uint32_t u32Access = 0;
4166	if (!pMixedCtx->ldtr.Attr.u)
4167	u32Access = X86DESCATTR_UNUSABLE;
4168	else
4169	u32Access = pMixedCtx->ldtr.Attr.u;
4170
4171	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4172	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4173	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4175
4176	/* Validate. */
4177	if (!(u32Access & X86DESCATTR_UNUSABLE))
4178	{
4179	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4180	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4181	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4182	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4183	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4184	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4185	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4186	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4187	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4188	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4189	}
4190
4191	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4192	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4193	}
4194
4195	/*
4196	* Guest IDTR.
4197	*/
4198	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4199	{
4200	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4201	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4202
4203	/* Validate. */
4204	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4205
4206	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4207	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4208	}
4209
4210	return VINF_SUCCESS;
4211	}
4212
4213
4214	/**
4215	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4216	* areas. These MSRs will automatically be loaded to the host CPU on every
4217	* successful VM entry and stored from the host CPU on every successful VM exit.
4218	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4219	*
4220	* @returns VBox status code.
4221	* @param pVCpu Pointer to the VMCPU.
4222	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4223	* out-of-sync. Make sure to update the required fields
4224	* before using them.
4225	*
4226	* @remarks No-long-jump zone!!!
4227	*/
4228	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4229	{
4230	AssertPtr(pVCpu);
4231	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4232
4233	/*
4234	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4235	*/
4236	int rc = VINF_SUCCESS;
4237	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4238	{
4239	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4240	PVM pVM = pVCpu->CTX_SUFF(pVM);
4241	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4242	uint32_t cGuestMsrs = 0;
4243
4244	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4245	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4246	* when the guest really is in 64-bit mode. */
4247	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4248	if (fSupportsLongMode)
4249	{
4250	pGuestMsr->u32Msr = MSR_K8_LSTAR;
4251	pGuestMsr->u32Reserved = 0;
4252	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4253	pGuestMsr++; cGuestMsrs++;
4254	pGuestMsr->u32Msr = MSR_K6_STAR;
4255	pGuestMsr->u32Reserved = 0;
4256	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4257	pGuestMsr++; cGuestMsrs++;
4258	pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4259	pGuestMsr->u32Reserved = 0;
4260	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4261	pGuestMsr++; cGuestMsrs++;
4262	pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4263	pGuestMsr->u32Reserved = 0;
4264	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4265	pGuestMsr++; cGuestMsrs++;
4266	}
4267
4268	/*
4269	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4270	* load the guest's copy always (since the MSR bitmap allows passthru unconditionally).
4271	*/
4272	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
4273	{
4274	pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4275	pGuestMsr->u32Reserved = 0;
4276	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4277	AssertRCReturn(rc, rc);
4278	pGuestMsr++; cGuestMsrs++;
4279	}
4280
4281	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4282	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4283	{
4284	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4285	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4286	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4287	}
4288
4289	/* Update the VCPU's copy of the guest MSR count. */
4290	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4291	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4292	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4293	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4294
4295	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4296	}
4297
4298	/*
4299	* Guest Sysenter MSRs.
4300	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4301	* VM-exits on WRMSRs for these MSRs.
4302	*/
4303	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4304	{
4305	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4306	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4307	}
4308
4309	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4310	{
4311	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4312	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4313	}
4314
4315	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4316	{
4317	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4318	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4319	}
4320
4321	return rc;
4322	}
4323
4324
4325	/**
4326	* Loads the guest activity state into the guest-state area in the VMCS.
4327	*
4328	* @returns VBox status code.
4329	* @param pVCpu Pointer to the VMCPU.
4330	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4331	* out-of-sync. Make sure to update the required fields
4332	* before using them.
4333	*
4334	* @remarks No-long-jump zone!!!
4335	*/
4336	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4337	{
4338	/** @todo See if we can make use of other states, e.g.
4339	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4340	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4341	{
4342	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4343	AssertRCReturn(rc, rc);
4344
4345	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4346	}
4347	return VINF_SUCCESS;
4348	}
4349
4350
4351	/**
4352	* Sets up the appropriate function to run guest code.
4353	*
4354	* @returns VBox status code.
4355	* @param pVCpu Pointer to the VMCPU.
4356	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4357	* out-of-sync. Make sure to update the required fields
4358	* before using them.
4359	*
4360	* @remarks No-long-jump zone!!!
4361	*/
4362	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4363	{
4364	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4365	{
4366	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4367	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4368	#endif
4369	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4370	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4371	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4372	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4373	{
4374	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4375	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4376	}
4377	#else
4378	/* 64-bit host or hybrid host. */
4379	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4380	#endif
4381	}
4382	else
4383	{
4384	/* Guest is not in long mode, use the 32-bit handler. */
4385	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4386	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4387	{
4388	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4389	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4390	}
4391	#else
4392	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4393	#endif
4394	}
4395	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4396	return VINF_SUCCESS;
4397	}
4398
4399
4400	/**
4401	* Wrapper for running the guest code in VT-x.
4402	*
4403	* @returns VBox strict status code.
4404	* @param pVM Pointer to the VM.
4405	* @param pVCpu Pointer to the VMCPU.
4406	* @param pCtx Pointer to the guest-CPU context.
4407	*
4408	* @remarks No-long-jump zone!!!
4409	*/
4410	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4411	{
4412	/*
4413	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4414	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4415	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4416	*/
4417	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4418	/** @todo Add stats for resume vs launch. */
4419	#ifdef VBOX_WITH_KERNEL_USING_XMM
4420	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4421	#else
4422	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4423	#endif
4424	}
4425
4426
4427	/**
4428	* Reports world-switch error and dumps some useful debug info.
4429	*
4430	* @param pVM Pointer to the VM.
4431	* @param pVCpu Pointer to the VMCPU.
4432	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4433	* @param pCtx Pointer to the guest-CPU context.
4434	* @param pVmxTransient Pointer to the VMX transient structure (only
4435	* exitReason updated).
4436	*/
4437	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4438	{
4439	Assert(pVM);
4440	Assert(pVCpu);
4441	Assert(pCtx);
4442	Assert(pVmxTransient);
4443	HMVMX_ASSERT_PREEMPT_SAFE();
4444
4445	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4446	switch (rcVMRun)
4447	{
4448	case VERR_VMX_INVALID_VMXON_PTR:
4449	AssertFailed();
4450	break;
4451	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4452	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4453	{
4454	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4455	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4456	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4457	AssertRC(rc);
4458
4459	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4460	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4461	Cannot do it here as we may have been long preempted. */
4462
4463	#ifdef VBOX_STRICT
4464	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4465	pVmxTransient->uExitReason));
4466	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4467	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4468	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4469	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4470	else
4471	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4472	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4473	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4474
4475	/* VMX control bits. */
4476	uint32_t u32Val;
4477	uint64_t u64Val;
4478	HMVMXHCUINTREG uHCReg;
4479	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4480	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4481	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4482	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4483	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4484	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4485	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4486	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4487	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4488	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4489	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4490	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4491	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4492	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4493	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4494	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4495	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4496	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4497	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4498	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4499	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4500	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4501	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4502	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4503	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4504	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4505	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4506	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4507	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4508	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4509	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4510	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4511	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4512	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4513	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4514	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4515	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4516	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4517	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4518	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4519	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4520	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4521
4522	/* Guest bits. */
4523	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4524	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4525	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4526	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4527	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4528	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4529	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4530	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4531
4532	/* Host bits. */
4533	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4534	Log4(("Host CR0 %#RHr\n", uHCReg));
4535	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4536	Log4(("Host CR3 %#RHr\n", uHCReg));
4537	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4538	Log4(("Host CR4 %#RHr\n", uHCReg));
4539
4540	RTGDTR HostGdtr;
4541	PCX86DESCHC pDesc;
4542	ASMGetGDTR(&HostGdtr);
4543	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4544	Log4(("Host CS %#08x\n", u32Val));
4545	if (u32Val < HostGdtr.cbGdt)
4546	{
4547	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4548	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4549	}
4550
4551	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4552	Log4(("Host DS %#08x\n", u32Val));
4553	if (u32Val < HostGdtr.cbGdt)
4554	{
4555	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4556	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4557	}
4558
4559	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4560	Log4(("Host ES %#08x\n", u32Val));
4561	if (u32Val < HostGdtr.cbGdt)
4562	{
4563	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4564	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4565	}
4566
4567	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4568	Log4(("Host FS %#08x\n", u32Val));
4569	if (u32Val < HostGdtr.cbGdt)
4570	{
4571	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4572	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4573	}
4574
4575	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4576	Log4(("Host GS %#08x\n", u32Val));
4577	if (u32Val < HostGdtr.cbGdt)
4578	{
4579	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4580	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4581	}
4582
4583	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4584	Log4(("Host SS %#08x\n", u32Val));
4585	if (u32Val < HostGdtr.cbGdt)
4586	{
4587	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4588	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4589	}
4590
4591	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4592	Log4(("Host TR %#08x\n", u32Val));
4593	if (u32Val < HostGdtr.cbGdt)
4594	{
4595	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4596	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4597	}
4598
4599	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4600	Log4(("Host TR Base %#RHv\n", uHCReg));
4601	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4602	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4603	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4604	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4605	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4606	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4607	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4608	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4609	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4610	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4611	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4612	Log4(("Host RSP %#RHv\n", uHCReg));
4613	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4614	Log4(("Host RIP %#RHv\n", uHCReg));
4615	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4616	if (HMVMX_IS_64BIT_HOST_MODE())
4617	{
4618	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4619	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4620	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4621	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4622	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4623	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4624	}
4625	# endif
4626	#endif /* VBOX_STRICT */
4627	break;
4628	}
4629
4630	default:
4631	/* Impossible */
4632	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4633	break;
4634	}
4635	NOREF(pVM);
4636	}
4637
4638
4639	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4640	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4641	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4642	#endif
4643	#ifdef VBOX_STRICT
4644	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4645	{
4646	switch (idxField)
4647	{
4648	case VMX_VMCS_GUEST_RIP:
4649	case VMX_VMCS_GUEST_RSP:
4650	case VMX_VMCS_GUEST_SYSENTER_EIP:
4651	case VMX_VMCS_GUEST_SYSENTER_ESP:
4652	case VMX_VMCS_GUEST_GDTR_BASE:
4653	case VMX_VMCS_GUEST_IDTR_BASE:
4654	case VMX_VMCS_GUEST_CS_BASE:
4655	case VMX_VMCS_GUEST_DS_BASE:
4656	case VMX_VMCS_GUEST_ES_BASE:
4657	case VMX_VMCS_GUEST_FS_BASE:
4658	case VMX_VMCS_GUEST_GS_BASE:
4659	case VMX_VMCS_GUEST_SS_BASE:
4660	case VMX_VMCS_GUEST_LDTR_BASE:
4661	case VMX_VMCS_GUEST_TR_BASE:
4662	case VMX_VMCS_GUEST_CR3:
4663	return true;
4664	}
4665	return false;
4666	}
4667
4668	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4669	{
4670	switch (idxField)
4671	{
4672	/* Read-only fields. */
4673	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4674	return true;
4675	}
4676	/* Remaining readable fields should also be writable. */
4677	return hmR0VmxIsValidWriteField(idxField);
4678	}
4679	#endif /* VBOX_STRICT */
4680
4681
4682	/**
4683	* Executes the specified handler in 64-bit mode.
4684	*
4685	* @returns VBox status code.
4686	* @param pVM Pointer to the VM.
4687	* @param pVCpu Pointer to the VMCPU.
4688	* @param pCtx Pointer to the guest CPU context.
4689	* @param enmOp The operation to perform.
4690	* @param cbParam Number of parameters.
4691	* @param paParam Array of 32-bit parameters.
4692	*/
4693	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4694	uint32_t *paParam)
4695	{
4696	int rc, rc2;
4697	PHMGLOBALCPUINFO pCpu;
4698	RTHCPHYS HCPhysCpuPage;
4699	RTCCUINTREG uOldEflags;
4700
4701	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4702	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4703	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4704	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4705
4706	#ifdef VBOX_STRICT
4707	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4708	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4709
4710	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4711	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4712	#endif
4713
4714	/* Disable interrupts. */
4715	uOldEflags = ASMIntDisableFlags();
4716
4717	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4718	RTCPUID idHostCpu = RTMpCpuId();
4719	CPUMR0SetLApic(pVCpu, idHostCpu);
4720	#endif
4721
4722	pCpu = HMR0GetCurrentCpu();
4723	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4724
4725	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4726	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4727
4728	/* Leave VMX Root Mode. */
4729	VMXDisable();
4730
4731	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4732
4733	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4734	CPUMSetHyperEIP(pVCpu, enmOp);
4735	for (int i = (int)cbParam - 1; i >= 0; i--)
4736	CPUMPushHyper(pVCpu, paParam[i]);
4737
4738	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4739
4740	/* Call the switcher. */
4741	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4742	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4743
4744	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4745	/* Make sure the VMX instructions don't cause #UD faults. */
4746	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4747
4748	/* Re-enter VMX Root Mode */
4749	rc2 = VMXEnable(HCPhysCpuPage);
4750	if (RT_FAILURE(rc2))
4751	{
4752	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4753	ASMSetFlags(uOldEflags);
4754	return rc2;
4755	}
4756
4757	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4758	AssertRC(rc2);
4759	Assert(!(ASMGetFlags() & X86_EFL_IF));
4760	ASMSetFlags(uOldEflags);
4761	return rc;
4762	}
4763
4764
4765	/**
4766	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4767	* supporting 64-bit guests.
4768	*
4769	* @returns VBox status code.
4770	* @param fResume Whether to VMLAUNCH or VMRESUME.
4771	* @param pCtx Pointer to the guest-CPU context.
4772	* @param pCache Pointer to the VMCS cache.
4773	* @param pVM Pointer to the VM.
4774	* @param pVCpu Pointer to the VMCPU.
4775	*/
4776	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4777	{
4778	uint32_t aParam[6];
4779	PHMGLOBALCPUINFO pCpu = NULL;
4780	RTHCPHYS HCPhysCpuPage = 0;
4781	int rc = VERR_INTERNAL_ERROR_5;
4782
4783	pCpu = HMR0GetCurrentCpu();
4784	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4785
4786	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4787	pCache->uPos = 1;
4788	pCache->interPD = PGMGetInterPaeCR3(pVM);
4789	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4790	#endif
4791
4792	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4793	pCache->TestIn.HCPhysCpuPage = 0;
4794	pCache->TestIn.HCPhysVmcs = 0;
4795	pCache->TestIn.pCache = 0;
4796	pCache->TestOut.HCPhysVmcs = 0;
4797	pCache->TestOut.pCache = 0;
4798	pCache->TestOut.pCtx = 0;
4799	pCache->TestOut.eflags = 0;
4800	#endif
4801
4802	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4803	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4804	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4805	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4806	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4807	aParam[5] = 0;
4808
4809	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4810	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4811	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4812	#endif
4813	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4814
4815	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4816	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4817	Assert(pCtx->dr[4] == 10);
4818	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4819	#endif
4820
4821	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4822	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4823	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4824	pVCpu->hm.s.vmx.HCPhysVmcs));
4825	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4826	pCache->TestOut.HCPhysVmcs));
4827	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4828	pCache->TestOut.pCache));
4829	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4830	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4831	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4832	pCache->TestOut.pCtx));
4833	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4834	#endif
4835	return rc;
4836	}
4837
4838
4839	/**
4840	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4841	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4842	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4843	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4844	*
4845	* @returns VBox status code.
4846	* @param pVM Pointer to the VM.
4847	* @param pVCpu Pointer to the VMCPU.
4848	*/
4849	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4850	{
4851	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4852	{ \
4853	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4854	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4855	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4856	++cReadFields; \
4857	}
4858
4859	AssertPtr(pVM);
4860	AssertPtr(pVCpu);
4861	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4862	uint32_t cReadFields = 0;
4863
4864	/*
4865	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4866	* and serve to indicate exceptions to the rules.
4867	*/
4868
4869	/* Guest-natural selector base fields. */
4870	#if 0
4871	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4872	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4873	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4874	#endif
4875	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4876	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4877	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4878	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4879	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4880	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4881	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4882	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4883	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4884	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4885	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4886	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4887	#if 0
4888	/* Unused natural width guest-state fields. */
4889	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4890	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4891	#endif
4892	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4893	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4894
4895	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4896	#if 0
4897	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4898	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4899	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4900	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4901	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4902	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4903	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4904	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4905	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4906	#endif
4907
4908	/* Natural width guest-state fields. */
4909	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4910	#if 0
4911	/* Currently unused field. */
4912	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4913	#endif
4914
4915	if (pVM->hm.s.fNestedPaging)
4916	{
4917	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4918	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4919	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4920	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4921	}
4922	else
4923	{
4924	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4925	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4926	}
4927
4928	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4929	return VINF_SUCCESS;
4930	}
4931
4932
4933	/**
4934	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4935	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4936	* darwin, running 64-bit guests).
4937	*
4938	* @returns VBox status code.
4939	* @param pVCpu Pointer to the VMCPU.
4940	* @param idxField The VMCS field encoding.
4941	* @param u64Val 16, 32 or 64 bits value.
4942	*/
4943	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4944	{
4945	int rc;
4946	switch (idxField)
4947	{
4948	/*
4949	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4950	*/
4951	/* 64-bit Control fields. */
4952	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4953	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4954	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4955	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4956	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4957	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4958	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4959	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4960	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4961	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4962	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4963	case VMX_VMCS64_CTRL_EPTP_FULL:
4964	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4965	/* 64-bit Guest-state fields. */
4966	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4967	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4968	case VMX_VMCS64_GUEST_PAT_FULL:
4969	case VMX_VMCS64_GUEST_EFER_FULL:
4970	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4971	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4972	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4973	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4974	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4975	/* 64-bit Host-state fields. */
4976	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4977	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4978	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4979	{
4980	rc = VMXWriteVmcs32(idxField, u64Val);
4981	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4982	break;
4983	}
4984
4985	/*
4986	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4987	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4988	*/
4989	/* Natural-width Guest-state fields. */
4990	case VMX_VMCS_GUEST_CR3:
4991	case VMX_VMCS_GUEST_ES_BASE:
4992	case VMX_VMCS_GUEST_CS_BASE:
4993	case VMX_VMCS_GUEST_SS_BASE:
4994	case VMX_VMCS_GUEST_DS_BASE:
4995	case VMX_VMCS_GUEST_FS_BASE:
4996	case VMX_VMCS_GUEST_GS_BASE:
4997	case VMX_VMCS_GUEST_LDTR_BASE:
4998	case VMX_VMCS_GUEST_TR_BASE:
4999	case VMX_VMCS_GUEST_GDTR_BASE:
5000	case VMX_VMCS_GUEST_IDTR_BASE:
5001	case VMX_VMCS_GUEST_RSP:
5002	case VMX_VMCS_GUEST_RIP:
5003	case VMX_VMCS_GUEST_SYSENTER_ESP:
5004	case VMX_VMCS_GUEST_SYSENTER_EIP:
5005	{
5006	if (!(u64Val >> 32))
5007	{
5008	/* If this field is 64-bit, VT-x will zero out the top bits. */
5009	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5010	}
5011	else
5012	{
5013	/* Assert that only the 32->64 switcher case should ever come here. */
5014	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5015	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5016	}
5017	break;
5018	}
5019
5020	default:
5021	{
5022	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5023	rc = VERR_INVALID_PARAMETER;
5024	break;
5025	}
5026	}
5027	AssertRCReturn(rc, rc);
5028	return rc;
5029	}
5030
5031
5032	/**
5033	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5034	* hosts (except darwin) for 64-bit guests.
5035	*
5036	* @param pVCpu Pointer to the VMCPU.
5037	* @param idxField The VMCS field encoding.
5038	* @param u64Val 16, 32 or 64 bits value.
5039	*/
5040	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5041	{
5042	AssertPtr(pVCpu);
5043	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5044
5045	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5046	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5047
5048	/* Make sure there are no duplicates. */
5049	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5050	{
5051	if (pCache->Write.aField[i] == idxField)
5052	{
5053	pCache->Write.aFieldVal[i] = u64Val;
5054	return VINF_SUCCESS;
5055	}
5056	}
5057
5058	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5059	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5060	pCache->Write.cValidEntries++;
5061	return VINF_SUCCESS;
5062	}
5063
5064	/* Enable later when the assembly code uses these as callbacks. */
5065	#if 0
5066	/*
5067	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5068	*
5069	* @param pVCpu Pointer to the VMCPU.
5070	* @param pCache Pointer to the VMCS cache.
5071	*
5072	* @remarks No-long-jump zone!!!
5073	*/
5074	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5075	{
5076	AssertPtr(pCache);
5077	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5078	{
5079	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5080	AssertRC(rc);
5081	}
5082	pCache->Write.cValidEntries = 0;
5083	}
5084
5085
5086	/**
5087	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5088	*
5089	* @param pVCpu Pointer to the VMCPU.
5090	* @param pCache Pointer to the VMCS cache.
5091	*
5092	* @remarks No-long-jump zone!!!
5093	*/
5094	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5095	{
5096	AssertPtr(pCache);
5097	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5098	{
5099	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5100	AssertRC(rc);
5101	}
5102	}
5103	#endif
5104	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5105
5106
5107	/**
5108	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5109	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5110	* timer.
5111	*
5112	* @returns VBox status code.
5113	* @param pVCpu Pointer to the VMCPU.
5114	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5115	* out-of-sync. Make sure to update the required fields
5116	* before using them.
5117	* @remarks No-long-jump zone!!!
5118	*/
5119	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5120	{
5121	int rc = VERR_INTERNAL_ERROR_5;
5122	bool fOffsettedTsc = false;
5123	PVM pVM = pVCpu->CTX_SUFF(pVM);
5124	if (pVM->hm.s.vmx.fUsePreemptTimer)
5125	{
5126	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5127
5128	/* Make sure the returned values have sane upper and lower boundaries. */
5129	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5130	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5131	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5132	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5133
5134	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5135	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5136	}
5137	else
5138	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5139
5140	if (fOffsettedTsc)
5141	{
5142	uint64_t u64CurTSC = ASMReadTSC();
5143	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5144	{
5145	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5146	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5147
5148	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5149	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5150	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5151	}
5152	else
5153	{
5154	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5155	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5156	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5157	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5158	}
5159	}
5160	else
5161	{
5162	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5163	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5164	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5165	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5166	}
5167	}
5168
5169
5170	/**
5171	* Determines if an exception is a contributory exception. Contributory
5172	* exceptions are ones which can cause double-faults. Page-fault is
5173	* intentionally not included here as it's a conditional contributory exception.
5174	*
5175	* @returns true if the exception is contributory, false otherwise.
5176	* @param uVector The exception vector.
5177	*/
5178	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5179	{
5180	switch (uVector)
5181	{
5182	case X86_XCPT_GP:
5183	case X86_XCPT_SS:
5184	case X86_XCPT_NP:
5185	case X86_XCPT_TS:
5186	case X86_XCPT_DE:
5187	return true;
5188	default:
5189	break;
5190	}
5191	return false;
5192	}
5193
5194
5195	/**
5196	* Sets an event as a pending event to be injected into the guest.
5197	*
5198	* @param pVCpu Pointer to the VMCPU.
5199	* @param u32IntInfo The VM-entry interruption-information field.
5200	* @param cbInstr The VM-entry instruction length in bytes (for software
5201	* interrupts, exceptions and privileged software
5202	* exceptions).
5203	* @param u32ErrCode The VM-entry exception error code.
5204	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5205	* page-fault.
5206	*
5207	* @remarks Statistics counter assumes this is a guest event being injected or
5208	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5209	* always incremented.
5210	*/
5211	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5212	RTGCUINTPTR GCPtrFaultAddress)
5213	{
5214	Assert(!pVCpu->hm.s.Event.fPending);
5215	pVCpu->hm.s.Event.fPending = true;
5216	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5217	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5218	pVCpu->hm.s.Event.cbInstr = cbInstr;
5219	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5220
5221	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5222	}
5223
5224
5225	/**
5226	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5227	*
5228	* @param pVCpu Pointer to the VMCPU.
5229	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5230	* out-of-sync. Make sure to update the required fields
5231	* before using them.
5232	*/
5233	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5234	{
5235	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5236	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5237	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5238	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5239	}
5240
5241
5242	/**
5243	* Handle a condition that occurred while delivering an event through the guest
5244	* IDT.
5245	*
5246	* @returns VBox status code (informational error codes included).
5247	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5248	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5249	* continue execution of the guest which will delivery the #DF.
5250	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5251	*
5252	* @param pVCpu Pointer to the VMCPU.
5253	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5254	* out-of-sync. Make sure to update the required fields
5255	* before using them.
5256	* @param pVmxTransient Pointer to the VMX transient structure.
5257	*
5258	* @remarks No-long-jump zone!!!
5259	*/
5260	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5261	{
5262	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5263	AssertRCReturn(rc, rc);
5264	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5265	{
5266	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
5267	AssertRCReturn(rc, rc);
5268
5269	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5270	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5271	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5272
5273	typedef enum
5274	{
5275	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5276	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5277	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5278	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5279	} VMXREFLECTXCPT;
5280
5281	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5282	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5283	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5284	{
5285	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5286	{
5287	enmReflect = VMXREFLECTXCPT_XCPT;
5288	#ifdef VBOX_STRICT
5289	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5290	&& uExitVector == X86_XCPT_PF)
5291	{
5292	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5293	}
5294	#endif
5295	if ( uExitVector == X86_XCPT_PF
5296	&& uIdtVector == X86_XCPT_PF)
5297	{
5298	pVmxTransient->fVectoringPF = true;
5299	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5300	}
5301	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5302	&& hmR0VmxIsContributoryXcpt(uExitVector)
5303	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5304	\|\| uIdtVector == X86_XCPT_PF))
5305	{
5306	enmReflect = VMXREFLECTXCPT_DF;
5307	}
5308	else if (uIdtVector == X86_XCPT_DF)
5309	enmReflect = VMXREFLECTXCPT_TF;
5310	}
5311	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5312	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5313	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5314	{
5315	/*
5316	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5317	* (whatever they are) as they reoccur when restarting the instruction.
5318	*/
5319	enmReflect = VMXREFLECTXCPT_XCPT;
5320	}
5321	}
5322	else
5323	{
5324	/*
5325	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5326	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5327	* original exception to the guest after handling the VM-exit.
5328	*/
5329	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5330	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5331	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5332	{
5333	enmReflect = VMXREFLECTXCPT_XCPT;
5334	}
5335	}
5336
5337	switch (enmReflect)
5338	{
5339	case VMXREFLECTXCPT_XCPT:
5340	{
5341	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5342	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5343	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5344
5345	uint32_t u32ErrCode = 0;
5346	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5347	{
5348	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5349	AssertRCReturn(rc, rc);
5350	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5351	}
5352
5353	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5354	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5355	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5356	rc = VINF_SUCCESS;
5357	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5358	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5359
5360	break;
5361	}
5362
5363	case VMXREFLECTXCPT_DF:
5364	{
5365	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5366	rc = VINF_HM_DOUBLE_FAULT;
5367	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5368	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5369
5370	break;
5371	}
5372
5373	case VMXREFLECTXCPT_TF:
5374	{
5375	rc = VINF_EM_RESET;
5376	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5377	uExitVector));
5378	break;
5379	}
5380
5381	default:
5382	Assert(rc == VINF_SUCCESS);
5383	break;
5384	}
5385	}
5386	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5387	return rc;
5388	}
5389
5390
5391	/**
5392	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5393	*
5394	* @returns VBox status code.
5395	* @param pVCpu Pointer to the VMCPU.
5396	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5397	* out-of-sync. Make sure to update the required fields
5398	* before using them.
5399	*
5400	* @remarks No-long-jump zone!!!
5401	*/
5402	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5403	{
5404	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5405	{
5406	uint32_t uVal = 0;
5407	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5408	AssertRCReturn(rc, rc);
5409
5410	uint32_t uShadow = 0;
5411	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5412	AssertRCReturn(rc, rc);
5413
5414	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5415	CPUMSetGuestCR0(pVCpu, uVal);
5416	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
5417	}
5418	return VINF_SUCCESS;
5419	}
5420
5421
5422	/**
5423	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5424	*
5425	* @returns VBox status code.
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	*
5431	* @remarks No-long-jump zone!!!
5432	*/
5433	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5434	{
5435	int rc = VINF_SUCCESS;
5436	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5437	{
5438	uint32_t uVal = 0;
5439	uint32_t uShadow = 0;
5440	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5441	AssertRCReturn(rc, rc);
5442	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5443	AssertRCReturn(rc, rc);
5444
5445	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5446	CPUMSetGuestCR4(pVCpu, uVal);
5447	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
5448	}
5449	return rc;
5450	}
5451
5452
5453	/**
5454	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5455	*
5456	* @returns VBox status code.
5457	* @param pVCpu Pointer to the VMCPU.
5458	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5459	* out-of-sync. Make sure to update the required fields
5460	* before using them.
5461	*
5462	* @remarks No-long-jump zone!!!
5463	*/
5464	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5465	{
5466	int rc = VINF_SUCCESS;
5467	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5468	{
5469	uint64_t u64Val = 0;
5470	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5471	AssertRCReturn(rc, rc);
5472
5473	pMixedCtx->rip = u64Val;
5474	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
5475	}
5476	return rc;
5477	}
5478
5479
5480	/**
5481	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5482	*
5483	* @returns VBox status code.
5484	* @param pVCpu Pointer to the VMCPU.
5485	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5486	* out-of-sync. Make sure to update the required fields
5487	* before using them.
5488	*
5489	* @remarks No-long-jump zone!!!
5490	*/
5491	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5492	{
5493	int rc = VINF_SUCCESS;
5494	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5495	{
5496	uint64_t u64Val = 0;
5497	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5498	AssertRCReturn(rc, rc);
5499
5500	pMixedCtx->rsp = u64Val;
5501	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
5502	}
5503	return rc;
5504	}
5505
5506
5507	/**
5508	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5509	*
5510	* @returns VBox status code.
5511	* @param pVCpu Pointer to the VMCPU.
5512	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5513	* out-of-sync. Make sure to update the required fields
5514	* before using them.
5515	*
5516	* @remarks No-long-jump zone!!!
5517	*/
5518	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5519	{
5520	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5521	{
5522	uint32_t uVal = 0;
5523	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5524	AssertRCReturn(rc, rc);
5525
5526	pMixedCtx->eflags.u32 = uVal;
5527	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5528	{
5529	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5530	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5531
5532	pMixedCtx->eflags.Bits.u1VM = 0;
5533	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5534	}
5535
5536	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5537	}
5538	return VINF_SUCCESS;
5539	}
5540
5541
5542	/**
5543	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5544	* guest-CPU context.
5545	*/
5546	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5547	{
5548	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5549	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5550	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5551	return rc;
5552	}
5553
5554
5555	/**
5556	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5557	* from the guest-state area in the VMCS.
5558	*
5559	* @param pVCpu Pointer to the VMCPU.
5560	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5561	* out-of-sync. Make sure to update the required fields
5562	* before using them.
5563	*
5564	* @remarks No-long-jump zone!!!
5565	*/
5566	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5567	{
5568	uint32_t uIntrState = 0;
5569	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5570	AssertRC(rc);
5571
5572	if (!uIntrState)
5573	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5574	else
5575	{
5576	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5577	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5578	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5579	AssertRC(rc);
5580	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5581	AssertRC(rc);
5582
5583	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5584	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5585	}
5586	}
5587
5588
5589	/**
5590	* Saves the guest's activity state.
5591	*
5592	* @returns VBox status code.
5593	* @param pVCpu Pointer to the VMCPU.
5594	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5595	* out-of-sync. Make sure to update the required fields
5596	* before using them.
5597	*
5598	* @remarks No-long-jump zone!!!
5599	*/
5600	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5601	{
5602	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5603	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5604	return VINF_SUCCESS;
5605	}
5606
5607
5608	/**
5609	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5610	* the current VMCS into the guest-CPU context.
5611	*
5612	* @returns VBox status code.
5613	* @param pVCpu Pointer to the VMCPU.
5614	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5615	* out-of-sync. Make sure to update the required fields
5616	* before using them.
5617	*
5618	* @remarks No-long-jump zone!!!
5619	*/
5620	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5621	{
5622	int rc = VINF_SUCCESS;
5623	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5624	{
5625	uint32_t u32Val = 0;
5626	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5627	pMixedCtx->SysEnter.cs = u32Val;
5628	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5629	}
5630
5631	uint64_t u64Val = 0;
5632	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5633	{
5634	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5635	pMixedCtx->SysEnter.eip = u64Val;
5636	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5637	}
5638	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5639	{
5640	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5641	pMixedCtx->SysEnter.esp = u64Val;
5642	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5643	}
5644	return rc;
5645	}
5646
5647
5648	/**
5649	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5650	* context.
5651	*
5652	* @returns VBox status code.
5653	* @param pVCpu Pointer to the VMCPU.
5654	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5655	* out-of-sync. Make sure to update the required fields
5656	* before using them.
5657	*
5658	* @remarks No-long-jump zone!!!
5659	*/
5660	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5661	{
5662	int rc = VINF_SUCCESS;
5663	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5664	{
5665	uint64_t u64Val = 0;
5666	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5667	pMixedCtx->fs.u64Base = u64Val;
5668	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5669	}
5670	return rc;
5671	}
5672
5673
5674	/**
5675	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5676	* context.
5677	*
5678	* @returns VBox status code.
5679	* @param pVCpu Pointer to the VMCPU.
5680	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5681	* out-of-sync. Make sure to update the required fields
5682	* before using them.
5683	*
5684	* @remarks No-long-jump zone!!!
5685	*/
5686	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5687	{
5688	int rc = VINF_SUCCESS;
5689	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5690	{
5691	uint64_t u64Val = 0;
5692	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5693	pMixedCtx->gs.u64Base = u64Val;
5694	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5695	}
5696	return rc;
5697	}
5698
5699
5700	/**
5701	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5702	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5703	* and TSC_AUX.
5704	*
5705	* @returns VBox status code.
5706	* @param pVCpu Pointer to the VMCPU.
5707	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5708	* out-of-sync. Make sure to update the required fields
5709	* before using them.
5710	*
5711	* @remarks No-long-jump zone!!!
5712	*/
5713	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5714	{
5715	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5716	return VINF_SUCCESS;
5717
5718	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5719	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5720	{
5721	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5722	pMsr += i;
5723	switch (pMsr->u32Msr)
5724	{
5725	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5726	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5727	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5728	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5729	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5730	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5731	default:
5732	{
5733	AssertFailed();
5734	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5735	}
5736	}
5737	}
5738	#endif
5739
5740	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5741	return VINF_SUCCESS;
5742	}
5743
5744
5745	/**
5746	* Saves the guest control registers from the current VMCS into the guest-CPU
5747	* context.
5748	*
5749	* @returns VBox status code.
5750	* @param pVCpu Pointer to the VMCPU.
5751	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5752	* out-of-sync. Make sure to update the required fields
5753	* before using them.
5754	*
5755	* @remarks No-long-jump zone!!!
5756	*/
5757	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5758	{
5759	/* Guest CR0. Guest FPU. */
5760	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5761	AssertRCReturn(rc, rc);
5762
5763	/* Guest CR4. */
5764	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5765	AssertRCReturn(rc, rc);
5766
5767	/* Guest CR2 - updated always during the world-switch or in #PF. */
5768	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5769	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5770	{
5771	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5772	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5773
5774	PVM pVM = pVCpu->CTX_SUFF(pVM);
5775	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5776	\|\| ( pVM->hm.s.fNestedPaging
5777	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5778	{
5779	uint64_t u64Val = 0;
5780	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5781	if (pMixedCtx->cr3 != u64Val)
5782	{
5783	CPUMSetGuestCR3(pVCpu, u64Val);
5784	if (VMMRZCallRing3IsEnabled(pVCpu))
5785	{
5786	PGMUpdateCR3(pVCpu, u64Val);
5787	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5788	}
5789	else
5790	{
5791	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5792	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5793	}
5794	}
5795
5796	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5797	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5798	{
5799	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5800	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5801	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5802	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5803
5804	if (VMMRZCallRing3IsEnabled(pVCpu))
5805	{
5806	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5807	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5808	}
5809	else
5810	{
5811	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5812	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5813	}
5814	}
5815	}
5816
5817	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5818	}
5819
5820	/*
5821	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5822	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5823	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5824	*
5825	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
5826	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
5827	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
5828	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
5829	*
5830	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5831	*/
5832	if (VMMRZCallRing3IsEnabled(pVCpu))
5833	{
5834	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5835	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5836
5837	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5838	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5839
5840	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5841	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5842	}
5843
5844	return rc;
5845	}
5846
5847
5848	/**
5849	* Reads a guest segment register from the current VMCS into the guest-CPU
5850	* context.
5851	*
5852	* @returns VBox status code.
5853	* @param pVCpu Pointer to the VMCPU.
5854	* @param idxSel Index of the selector in the VMCS.
5855	* @param idxLimit Index of the segment limit in the VMCS.
5856	* @param idxBase Index of the segment base in the VMCS.
5857	* @param idxAccess Index of the access rights of the segment in the VMCS.
5858	* @param pSelReg Pointer to the segment selector.
5859	*
5860	* @remarks No-long-jump zone!!!
5861	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5862	* macro as that takes care of whether to read from the VMCS cache or
5863	* not.
5864	*/
5865	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5866	PCPUMSELREG pSelReg)
5867	{
5868	uint32_t u32Val = 0;
5869	int rc = VMXReadVmcs32(idxSel, &u32Val);
5870	AssertRCReturn(rc, rc);
5871	pSelReg->Sel = (uint16_t)u32Val;
5872	pSelReg->ValidSel = (uint16_t)u32Val;
5873	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5874
5875	rc = VMXReadVmcs32(idxLimit, &u32Val);
5876	AssertRCReturn(rc, rc);
5877	pSelReg->u32Limit = u32Val;
5878
5879	uint64_t u64Val = 0;
5880	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5881	AssertRCReturn(rc, rc);
5882	pSelReg->u64Base = u64Val;
5883
5884	rc = VMXReadVmcs32(idxAccess, &u32Val);
5885	AssertRCReturn(rc, rc);
5886	pSelReg->Attr.u = u32Val;
5887
5888	/*
5889	* If VT-x marks the segment as unusable, most other bits remain undefined:
5890	* - For CS the L, D and G bits have meaning.
5891	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5892	* - For the remaining data segments no bits are defined.
5893	*
5894	* The present bit and the unusable bit has been observed to be set at the
5895	* same time (the selector was supposed to invalid as we started executing
5896	* a V8086 interrupt in ring-0).
5897	*
5898	* What should be important for the rest of the VBox code that the P bit is
5899	* cleared. Some of the other VBox code recognizes the unusable bit, but
5900	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
5901	* safe side here, we'll strip off P and other bits we don't care about. If
5902	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5903	*
5904	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5905	*/
5906	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5907	{
5908	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5909
5910	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5911	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
5912	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
5913
5914	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5915	#ifdef DEBUG_bird
5916	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5917	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5918	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5919	#endif
5920	}
5921	return VINF_SUCCESS;
5922	}
5923
5924
5925	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5926	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5927	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5928	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5929	#else
5930	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5931	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5932	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5933	#endif
5934
5935
5936	/**
5937	* Saves the guest segment registers from the current VMCS into the guest-CPU
5938	* context.
5939	*
5940	* @returns VBox status code.
5941	* @param pVCpu Pointer to the VMCPU.
5942	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5943	* out-of-sync. Make sure to update the required fields
5944	* before using them.
5945	*
5946	* @remarks No-long-jump zone!!!
5947	*/
5948	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5949	{
5950	/* Guest segment registers. */
5951	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5952	{
5953	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5954	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5955	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5956	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5957	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5958	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5959	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5960
5961	/* Restore segment attributes for real-on-v86 mode hack. */
5962	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5963	{
5964	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5965	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5966	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5967	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5968	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5969	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5970	}
5971	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5972	}
5973
5974	return VINF_SUCCESS;
5975	}
5976
5977
5978	/**
5979	* Saves the guest descriptor table registers and task register from the current
5980	* VMCS into the guest-CPU context.
5981	*
5982	* @returns VBox status code.
5983	* @param pVCpu Pointer to the VMCPU.
5984	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5985	* out-of-sync. Make sure to update the required fields
5986	* before using them.
5987	*
5988	* @remarks No-long-jump zone!!!
5989	*/
5990	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5991	{
5992	int rc = VINF_SUCCESS;
5993
5994	/* Guest LDTR. */
5995	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5996	{
5997	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5998	AssertRCReturn(rc, rc);
5999	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
6000	}
6001
6002	/* Guest GDTR. */
6003	uint64_t u64Val = 0;
6004	uint32_t u32Val = 0;
6005	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
6006	{
6007	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6008	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6009	pMixedCtx->gdtr.pGdt = u64Val;
6010	pMixedCtx->gdtr.cbGdt = u32Val;
6011	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
6012	}
6013
6014	/* Guest IDTR. */
6015	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
6016	{
6017	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6018	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6019	pMixedCtx->idtr.pIdt = u64Val;
6020	pMixedCtx->idtr.cbIdt = u32Val;
6021	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
6022	}
6023
6024	/* Guest TR. */
6025	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
6026	{
6027	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6028	AssertRCReturn(rc, rc);
6029
6030	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6031	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6032	{
6033	rc = VMXLOCAL_READ_SEG(TR, tr);
6034	AssertRCReturn(rc, rc);
6035	}
6036	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
6037	}
6038	return rc;
6039	}
6040
6041	#undef VMXLOCAL_READ_SEG
6042
6043
6044	/**
6045	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6046	* context.
6047	*
6048	* @returns VBox status code.
6049	* @param pVCpu Pointer to the VMCPU.
6050	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6051	* out-of-sync. Make sure to update the required fields
6052	* before using them.
6053	*
6054	* @remarks No-long-jump zone!!!
6055	*/
6056	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6057	{
6058	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
6059	{
6060	if (!pVCpu->hm.s.fUsingHyperDR7)
6061	{
6062	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6063	uint32_t u32Val;
6064	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6065	pMixedCtx->dr[7] = u32Val;
6066	}
6067
6068	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
6069	}
6070	return VINF_SUCCESS;
6071	}
6072
6073
6074	/**
6075	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6076	*
6077	* @returns VBox status code.
6078	* @param pVCpu Pointer to the VMCPU.
6079	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6080	* out-of-sync. Make sure to update the required fields
6081	* before using them.
6082	*
6083	* @remarks No-long-jump zone!!!
6084	*/
6085	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6086	{
6087	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6088	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
6089	return VINF_SUCCESS;
6090	}
6091
6092
6093	/**
6094	* Saves the entire guest state from the currently active VMCS into the
6095	* guest-CPU context. This essentially VMREADs all guest-data.
6096	*
6097	* @returns VBox status code.
6098	* @param pVCpu Pointer to the VMCPU.
6099	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6100	* out-of-sync. Make sure to update the required fields
6101	* before using them.
6102	*/
6103	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6104	{
6105	Assert(pVCpu);
6106	Assert(pMixedCtx);
6107
6108	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
6109	return VINF_SUCCESS;
6110
6111	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6112	again on the ring-3 callback path, there is no real need to. */
6113	if (VMMRZCallRing3IsEnabled(pVCpu))
6114	VMMR0LogFlushDisable(pVCpu);
6115	else
6116	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6117	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6118
6119	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6120	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6121
6122	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6123	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6124
6125	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6126	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6127
6128	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6129	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6130
6131	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6132	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6133
6134	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6135	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6136
6137	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
6138	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6139
6140	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
6141	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6142
6143	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6144	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6145
6146	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6147	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6148
6149	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6150	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6151
6152	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
6153	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
6154
6155	if (VMMRZCallRing3IsEnabled(pVCpu))
6156	VMMR0LogFlushEnable(pVCpu);
6157
6158	return rc;
6159	}
6160
6161
6162	/**
6163	* Check per-VM and per-VCPU force flag actions that require us to go back to
6164	* ring-3 for one reason or another.
6165	*
6166	* @returns VBox status code (information status code included).
6167	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6168	* ring-3.
6169	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6170	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6171	* interrupts)
6172	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6173	* all EMTs to be in ring-3.
6174	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6175	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6176	* to the EM loop.
6177	*
6178	* @param pVM Pointer to the VM.
6179	* @param pVCpu Pointer to the VMCPU.
6180	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6181	* out-of-sync. Make sure to update the required fields
6182	* before using them.
6183	*/
6184	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6185	{
6186	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6187
6188	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6189	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6190	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6191	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6192	{
6193	/* We need the control registers now, make sure the guest-CPU context is updated. */
6194	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6195	AssertRCReturn(rc3, rc3);
6196
6197	/* Pending HM CR3 sync. */
6198	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6199	{
6200	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6201	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6202	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6203	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6204	}
6205
6206	/* Pending HM PAE PDPEs. */
6207	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6208	{
6209	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6210	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6211	}
6212
6213	/* Pending PGM C3 sync. */
6214	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6215	{
6216	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6217	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6218	if (rc2 != VINF_SUCCESS)
6219	{
6220	AssertRC(rc2);
6221	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6222	return rc2;
6223	}
6224	}
6225
6226	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6227	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6228	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6229	{
6230	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6231	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6232	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6233	return rc2;
6234	}
6235
6236	/* Pending VM request packets, such as hardware interrupts. */
6237	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6238	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6239	{
6240	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6241	return VINF_EM_PENDING_REQUEST;
6242	}
6243
6244	/* Pending PGM pool flushes. */
6245	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6246	{
6247	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6248	return VINF_PGM_POOL_FLUSH_PENDING;
6249	}
6250
6251	/* Pending DMA requests. */
6252	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6253	{
6254	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6255	return VINF_EM_RAW_TO_R3;
6256	}
6257	}
6258
6259	return VINF_SUCCESS;
6260	}
6261
6262
6263	/**
6264	* Converts any TRPM trap into a pending HM event. This is typically used when
6265	* entering from ring-3 (not longjmp returns).
6266	*
6267	* @param pVCpu Pointer to the VMCPU.
6268	*/
6269	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6270	{
6271	Assert(TRPMHasTrap(pVCpu));
6272	Assert(!pVCpu->hm.s.Event.fPending);
6273
6274	uint8_t uVector;
6275	TRPMEVENT enmTrpmEvent;
6276	RTGCUINT uErrCode;
6277	RTGCUINTPTR GCPtrFaultAddress;
6278	uint8_t cbInstr;
6279
6280	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6281	AssertRC(rc);
6282
6283	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6284	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6285	if (enmTrpmEvent == TRPM_TRAP)
6286	{
6287	switch (uVector)
6288	{
6289	case X86_XCPT_BP:
6290	case X86_XCPT_OF:
6291	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6292	break;
6293
6294	case X86_XCPT_PF:
6295	case X86_XCPT_DF:
6296	case X86_XCPT_TS:
6297	case X86_XCPT_NP:
6298	case X86_XCPT_SS:
6299	case X86_XCPT_GP:
6300	case X86_XCPT_AC:
6301	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6302	/* no break! */
6303	default:
6304	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6305	break;
6306	}
6307	}
6308	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6309	{
6310	if (uVector == X86_XCPT_NMI)
6311	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6312	else
6313	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6314	}
6315	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6316	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6317	else
6318	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6319
6320	rc = TRPMResetTrap(pVCpu);
6321	AssertRC(rc);
6322	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6323	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6324
6325	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6326	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6327	}
6328
6329
6330	/**
6331	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6332	* VT-x to execute any instruction.
6333	*
6334	* @param pvCpu Pointer to the VMCPU.
6335	*/
6336	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6337	{
6338	Assert(pVCpu->hm.s.Event.fPending);
6339
6340	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6341	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6342	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6343	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6344
6345	/* If a trap was already pending, we did something wrong! */
6346	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6347
6348	TRPMEVENT enmTrapType;
6349	switch (uVectorType)
6350	{
6351	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6352	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6353	enmTrapType = TRPM_HARDWARE_INT;
6354	break;
6355
6356	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6357	enmTrapType = TRPM_SOFTWARE_INT;
6358	break;
6359
6360	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6361	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6362	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6363	enmTrapType = TRPM_TRAP;
6364	break;
6365
6366	default:
6367	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6368	enmTrapType = TRPM_32BIT_HACK;
6369	break;
6370	}
6371
6372	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6373
6374	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6375	AssertRC(rc);
6376
6377	if (fErrorCodeValid)
6378	TRPMSetErrorCode(pVCpu, uErrorCode);
6379
6380	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6381	&& uVector == X86_XCPT_PF)
6382	{
6383	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6384	}
6385	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6386	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6387	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6388	{
6389	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6390	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6391	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6392	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6393	}
6394	pVCpu->hm.s.Event.fPending = false;
6395	}
6396
6397
6398	/**
6399	* Does the necessary state syncing before returning to ring-3 for any reason
6400	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6401	*
6402	* @returns VBox status code.
6403	* @param pVM Pointer to the VM.
6404	* @param pVCpu Pointer to the VMCPU.
6405	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6406	* be out-of-sync. Make sure to update the required
6407	* fields before using them.
6408	* @param fSaveGuestState Whether to save the guest state or not.
6409	*
6410	* @remarks If you modify code here, make sure to check whether
6411	* hmR0VmxCallRing3Callback() needs to be updated too.
6412	* @remarks No-long-jmp zone!!!
6413	*/
6414	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6415	{
6416	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6417	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6418
6419	RTCPUID idCpu = RTMpCpuId();
6420	Log4Func(("HostCpuId=%u\n", idCpu));
6421
6422	/* Save the guest state if necessary. */
6423	if ( fSaveGuestState
6424	&& pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6425	{
6426	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6427	AssertRCReturn(rc, rc);
6428	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6429	}
6430
6431	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6432	if (CPUMIsGuestFPUStateActive(pVCpu))
6433	{
6434	/* We shouldn't reload CR0 without saving it first. */
6435	if (!fSaveGuestState)
6436	{
6437	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6438	AssertRCReturn(rc, rc);
6439	}
6440	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6441	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6442	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6443	}
6444
6445	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6446	#ifdef VBOX_STRICT
6447	if (CPUMIsHyperDebugStateActive(pVCpu))
6448	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6449	#endif
6450	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6451	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6452	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6453	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6454
6455	#if HC_ARCH_BITS == 64
6456	/* Restore host-state bits that VT-x only restores partially. */
6457	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6458	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6459	{
6460	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6461	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6462	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6463	}
6464	#endif
6465
6466	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6467	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6468	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6469	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6470	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6471	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6472	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6473	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6474
6475	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6476
6477	/** @todo This kinda defeats the purpose of having preemption hooks.
6478	* The problem is, deregistering the hooks should be moved to a place that
6479	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6480	* context.
6481	*/
6482	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6483	{
6484	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6485	AssertRCReturn(rc, rc);
6486
6487	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6488	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6489	}
6490	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6491	NOREF(idCpu);
6492
6493	return VINF_SUCCESS;
6494	}
6495
6496
6497	/**
6498	* Leaves the VT-x session.
6499	*
6500	* @returns VBox status code.
6501	* @param pVM Pointer to the VM.
6502	* @param pVCpu Pointer to the VMCPU.
6503	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6504	* out-of-sync. Make sure to update the required fields
6505	* before using them.
6506	*
6507	* @remarks No-long-jmp zone!!!
6508	*/
6509	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6510	{
6511	HM_DISABLE_PREEMPT_IF_NEEDED();
6512	HMVMX_ASSERT_CPU_SAFE();
6513	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6514	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6515
6516	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6517	and done this from the VMXR0ThreadCtxCallback(). */
6518	if (!pVCpu->hm.s.fLeaveDone)
6519	{
6520	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6521	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6522	pVCpu->hm.s.fLeaveDone = true;
6523	}
6524
6525	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6526	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6527	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6528	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6529	VMMR0ThreadCtxHooksDeregister(pVCpu);
6530
6531	/* Leave HM context. This takes care of local init (term). */
6532	int rc = HMR0LeaveCpu(pVCpu);
6533
6534	HM_RESTORE_PREEMPT_IF_NEEDED();
6535
6536	return rc;
6537	}
6538
6539
6540	/**
6541	* Does the necessary state syncing before doing a longjmp to ring-3.
6542	*
6543	* @returns VBox status code.
6544	* @param pVM Pointer to the VM.
6545	* @param pVCpu Pointer to the VMCPU.
6546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6547	* out-of-sync. Make sure to update the required fields
6548	* before using them.
6549	*
6550	* @remarks No-long-jmp zone!!!
6551	*/
6552	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6553	{
6554	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6555	}
6556
6557
6558	/**
6559	* Take necessary actions before going back to ring-3.
6560	*
6561	* An action requires us to go back to ring-3. This function does the necessary
6562	* steps before we can safely return to ring-3. This is not the same as longjmps
6563	* to ring-3, this is voluntary and prepares the guest so it may continue
6564	* executing outside HM (recompiler/IEM).
6565	*
6566	* @returns VBox status code.
6567	* @param pVM Pointer to the VM.
6568	* @param pVCpu Pointer to the VMCPU.
6569	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6570	* out-of-sync. Make sure to update the required fields
6571	* before using them.
6572	* @param rcExit The reason for exiting to ring-3. Can be
6573	* VINF_VMM_UNKNOWN_RING3_CALL.
6574	*/
6575	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6576	{
6577	Assert(pVM);
6578	Assert(pVCpu);
6579	Assert(pMixedCtx);
6580	HMVMX_ASSERT_PREEMPT_SAFE();
6581
6582	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6583	{
6584	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6585	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6586	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6587	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6588	}
6589
6590	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6591	VMMRZCallRing3Disable(pVCpu);
6592	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6593
6594	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6595	if (pVCpu->hm.s.Event.fPending)
6596	{
6597	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6598	Assert(!pVCpu->hm.s.Event.fPending);
6599	}
6600
6601	/* Save guest state and restore host state bits. */
6602	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6603	AssertRCReturn(rc, rc);
6604	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6605
6606	/* Sync recompiler state. */
6607	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6608	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6609	\| CPUM_CHANGED_LDTR
6610	\| CPUM_CHANGED_GDTR
6611	\| CPUM_CHANGED_IDTR
6612	\| CPUM_CHANGED_TR
6613	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6614	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6615	if ( pVM->hm.s.fNestedPaging
6616	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6617	{
6618	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6619	}
6620
6621	Assert(!pVCpu->hm.s.fClearTrapFlag);
6622
6623	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6624	if (rcExit != VINF_EM_RAW_INTERRUPT)
6625	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6626
6627	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6628
6629	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6630	VMMRZCallRing3RemoveNotification(pVCpu);
6631	VMMRZCallRing3Enable(pVCpu);
6632
6633	return rc;
6634	}
6635
6636
6637	/**
6638	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6639	* longjump to ring-3 and possibly get preempted.
6640	*
6641	* @returns VBox status code.
6642	* @param pVCpu Pointer to the VMCPU.
6643	* @param enmOperation The operation causing the ring-3 longjump.
6644	* @param pvUser Opaque pointer to the guest-CPU context. The data
6645	* may be out-of-sync. Make sure to update the required
6646	* fields before using them.
6647	* @remarks If you modify code here, make sure to check whether
6648	* hmR0VmxLeave() needs to be updated too.
6649	*/
6650	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6651	{
6652	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6653	{
6654	VMMRZCallRing3RemoveNotification(pVCpu);
6655	HM_DISABLE_PREEMPT_IF_NEEDED();
6656
6657	/* If anything here asserts or fails, good luck. */
6658	if (CPUMIsGuestFPUStateActive(pVCpu))
6659	CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6660
6661	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6662
6663	#if HC_ARCH_BITS == 64
6664	/* Restore host-state bits that VT-x only restores partially. */
6665	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6666	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6667	{
6668	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6669	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6670	}
6671	#endif
6672	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6673	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6674	{
6675	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6676	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6677	}
6678
6679	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6680	VMMR0ThreadCtxHooksDeregister(pVCpu);
6681
6682	HMR0LeaveCpu(pVCpu);
6683	HM_RESTORE_PREEMPT_IF_NEEDED();
6684	return VINF_SUCCESS;
6685	}
6686
6687	Assert(pVCpu);
6688	Assert(pvUser);
6689	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6690	HMVMX_ASSERT_PREEMPT_SAFE();
6691
6692	VMMRZCallRing3Disable(pVCpu);
6693	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6694
6695	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6696	enmOperation));
6697
6698	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6699	AssertRCReturn(rc, rc);
6700
6701	VMMRZCallRing3Enable(pVCpu);
6702	return VINF_SUCCESS;
6703	}
6704
6705
6706	/**
6707	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6708	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6709	*
6710	* @param pVCpu Pointer to the VMCPU.
6711	*/
6712	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6713	{
6714	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6715	{
6716	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6717	{
6718	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6719	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6720	AssertRC(rc);
6721	}
6722	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6723	}
6724
6725
6726	/**
6727	* Evaluates the event to be delivered to the guest and sets it as the pending
6728	* event.
6729	*
6730	* @param pVCpu Pointer to the VMCPU.
6731	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6732	* out-of-sync. Make sure to update the required fields
6733	* before using them.
6734	*/
6735	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6736	{
6737	Assert(!pVCpu->hm.s.Event.fPending);
6738
6739	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6740	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6741	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6742	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6743
6744	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6745	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6746	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6747	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6748	Assert(!TRPMHasTrap(pVCpu));
6749
6750	/** @todo SMI. SMIs take priority over NMIs. */
6751	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6752	{
6753	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6754	if ( !fBlockMovSS
6755	&& !fBlockSti)
6756	{
6757	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6758	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6759	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6760	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6761
6762	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6763	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6764	}
6765	else
6766	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6767	}
6768	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6769	&& !pVCpu->hm.s.fSingleInstruction)
6770	{
6771	/*
6772	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6773	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6774	* evaluated here and not set as pending, solely based on the force-flags.
6775	*/
6776	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6777	AssertRC(rc);
6778	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6779	if ( !fBlockInt
6780	&& !fBlockSti
6781	&& !fBlockMovSS)
6782	{
6783	uint8_t u8Interrupt;
6784	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6785	if (RT_SUCCESS(rc))
6786	{
6787	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6788	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6789	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6790
6791	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6792	}
6793	else
6794	{
6795	/** @todo Does this actually happen? If not turn it into an assertion. */
6796	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6797	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6798	}
6799	}
6800	else
6801	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6802	}
6803	}
6804
6805
6806	/**
6807	* Injects any pending events into the guest if the guest is in a state to
6808	* receive them.
6809	*
6810	* @returns VBox status code (informational status codes included).
6811	* @param pVCpu Pointer to the VMCPU.
6812	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6813	* out-of-sync. Make sure to update the required fields
6814	* before using them.
6815	*/
6816	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6817	{
6818	HMVMX_ASSERT_PREEMPT_SAFE();
6819	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6820
6821	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6822	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6823	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6824	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6825
6826	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6827	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6828	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6829	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6830	Assert(!TRPMHasTrap(pVCpu));
6831
6832	int rc = VINF_SUCCESS;
6833	if (pVCpu->hm.s.Event.fPending)
6834	{
6835	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
6836	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6837	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6838	{
6839	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6840	AssertRCReturn(rc, rc);
6841	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6842	Assert(!fBlockInt);
6843	Assert(!fBlockSti);
6844	Assert(!fBlockMovSS);
6845	}
6846	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6847	{
6848	Assert(!fBlockSti);
6849	Assert(!fBlockMovSS);
6850	}
6851	#endif
6852	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
6853	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
6854	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6855	AssertRCReturn(rc, rc);
6856
6857	/* Update the interruptibility-state as it could have been changed by
6858	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6859	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6860	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6861
6862	#ifdef VBOX_WITH_STATISTICS
6863	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6864	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6865	else
6866	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6867	#endif
6868	}
6869
6870	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6871	int rc2 = VINF_SUCCESS;
6872	if ( fBlockSti
6873	\|\| fBlockMovSS)
6874	{
6875	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6876	{
6877	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6878	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6879	{
6880	/*
6881	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6882	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6883	* See Intel spec. 27.3.4 "Saving Non-Register State".
6884	*/
6885	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6886	AssertRCReturn(rc, rc);
6887	}
6888	}
6889	else
6890	{
6891	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6892	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6893	uIntrState = 0;
6894	}
6895	}
6896
6897	/*
6898	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6899	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6900	*/
6901	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6902	AssertRC(rc2);
6903
6904	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6905	return rc;
6906	}
6907
6908
6909	/**
6910	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6911	*
6912	* @param pVCpu Pointer to the VMCPU.
6913	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6914	* out-of-sync. Make sure to update the required fields
6915	* before using them.
6916	*/
6917	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6918	{
6919	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6920	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6921	}
6922
6923
6924	/**
6925	* Injects a double-fault (#DF) exception into the VM.
6926	*
6927	* @returns VBox status code (informational status code included).
6928	* @param pVCpu Pointer to the VMCPU.
6929	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6930	* out-of-sync. Make sure to update the required fields
6931	* before using them.
6932	*/
6933	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6934	{
6935	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6936	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6937	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6938	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6939	puIntrState);
6940	}
6941
6942
6943	/**
6944	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6945	*
6946	* @param pVCpu Pointer to the VMCPU.
6947	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6948	* out-of-sync. Make sure to update the required fields
6949	* before using them.
6950	*/
6951	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6952	{
6953	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6954	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6955	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6956	}
6957
6958
6959	/**
6960	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6961	*
6962	* @param pVCpu Pointer to the VMCPU.
6963	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6964	* out-of-sync. Make sure to update the required fields
6965	* before using them.
6966	* @param cbInstr The value of RIP that is to be pushed on the guest
6967	* stack.
6968	*/
6969	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6970	{
6971	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6972	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6973	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6974	}
6975
6976
6977	/**
6978	* Injects a general-protection (#GP) fault into the VM.
6979	*
6980	* @returns VBox status code (informational status code included).
6981	* @param pVCpu Pointer to the VMCPU.
6982	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6983	* out-of-sync. Make sure to update the required fields
6984	* before using them.
6985	* @param u32ErrorCode The error code associated with the #GP.
6986	*/
6987	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6988	uint32_t *puIntrState)
6989	{
6990	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6991	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6992	if (fErrorCodeValid)
6993	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6994	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6995	puIntrState);
6996	}
6997
6998
6999	/**
7000	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7001	*
7002	* @param pVCpu Pointer to the VMCPU.
7003	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7004	* out-of-sync. Make sure to update the required fields
7005	* before using them.
7006	* @param uVector The software interrupt vector number.
7007	* @param cbInstr The value of RIP that is to be pushed on the guest
7008	* stack.
7009	*/
7010	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7011	{
7012	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7013	if ( uVector == X86_XCPT_BP
7014	\|\| uVector == X86_XCPT_OF)
7015	{
7016	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7017	}
7018	else
7019	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7020	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7021	}
7022
7023
7024	/**
7025	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7026	* stack.
7027	*
7028	* @returns VBox status code (information status code included).
7029	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7030	* @param pVM Pointer to the VM.
7031	* @param pMixedCtx Pointer to the guest-CPU context.
7032	* @param uValue The value to push to the guest stack.
7033	*/
7034	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7035	{
7036	/*
7037	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7038	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7039	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7040	*/
7041	if (pMixedCtx->sp == 1)
7042	return VINF_EM_RESET;
7043	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7044	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7045	AssertRCReturn(rc, rc);
7046	return rc;
7047	}
7048
7049
7050	/**
7051	* Injects an event into the guest upon VM-entry by updating the relevant fields
7052	* in the VM-entry area in the VMCS.
7053	*
7054	* @returns VBox status code (informational error codes included).
7055	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7056	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7057	*
7058	* @param pVCpu Pointer to the VMCPU.
7059	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7060	* be out-of-sync. Make sure to update the required
7061	* fields before using them.
7062	* @param u64IntInfo The VM-entry interruption-information field.
7063	* @param cbInstr The VM-entry instruction length in bytes (for
7064	* software interrupts, exceptions and privileged
7065	* software exceptions).
7066	* @param u32ErrCode The VM-entry exception error code.
7067	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7068	* @param puIntrState Pointer to the current guest interruptibility-state.
7069	* This interruptibility-state will be updated if
7070	* necessary. This cannot not be NULL.
7071	*
7072	* @remarks Requires CR0!
7073	* @remarks No-long-jump zone!!!
7074	*/
7075	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7076	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7077	{
7078	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7079	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7080	Assert(puIntrState);
7081	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7082
7083	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7084	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7085
7086	#ifdef VBOX_STRICT
7087	/* Validate the error-code-valid bit for hardware exceptions. */
7088	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7089	{
7090	switch (uVector)
7091	{
7092	case X86_XCPT_PF:
7093	case X86_XCPT_DF:
7094	case X86_XCPT_TS:
7095	case X86_XCPT_NP:
7096	case X86_XCPT_SS:
7097	case X86_XCPT_GP:
7098	case X86_XCPT_AC:
7099	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7100	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7101	/* fallthru */
7102	default:
7103	break;
7104	}
7105	}
7106	#endif
7107
7108	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7109	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7110	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7111
7112	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7113
7114	/* We require CR0 to check if the guest is in real-mode. */
7115	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7116	AssertRCReturn(rc, rc);
7117
7118	/*
7119	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7120	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7121	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7122	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7123	*/
7124	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7125	{
7126	PVM pVM = pVCpu->CTX_SUFF(pVM);
7127	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7128	{
7129	Assert(PDMVmmDevHeapIsEnabled(pVM));
7130	Assert(pVM->hm.s.vmx.pRealModeTSS);
7131
7132	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7133	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7134	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7135	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7136	AssertRCReturn(rc, rc);
7137	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
7138
7139	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7140	const size_t cbIdtEntry = sizeof(X86IDTR16);
7141	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7142	{
7143	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7144	if (uVector == X86_XCPT_DF)
7145	return VINF_EM_RESET;
7146	else if (uVector == X86_XCPT_GP)
7147	{
7148	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7149	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7150	}
7151
7152	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7153	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7154	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7155	}
7156
7157	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7158	uint16_t uGuestIp = pMixedCtx->ip;
7159	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7160	{
7161	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7162	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7163	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7164	}
7165	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7166	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7167
7168	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7169	X86IDTR16 IdtEntry;
7170	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7171	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7172	AssertRCReturn(rc, rc);
7173
7174	/* Construct the stack frame for the interrupt/exception handler. */
7175	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7176	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7177	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7178	AssertRCReturn(rc, rc);
7179
7180	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7181	if (rc == VINF_SUCCESS)
7182	{
7183	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7184	pMixedCtx->rip = IdtEntry.offSel;
7185	pMixedCtx->cs.Sel = IdtEntry.uSel;
7186	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7187	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7188	&& uVector == X86_XCPT_PF)
7189	{
7190	pMixedCtx->cr2 = GCPtrFaultAddress;
7191	}
7192
7193	/* If any other guest-state bits are changed here, make sure to update
7194	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7195	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7196	\| HM_CHANGED_GUEST_RIP
7197	\| HM_CHANGED_GUEST_RFLAGS
7198	\| HM_CHANGED_GUEST_RSP);
7199
7200	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7201	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7202	{
7203	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7204	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7205	Log4(("Clearing inhibition due to STI.\n"));
7206	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7207	}
7208	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7209
7210	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7211	it, if we are returning to ring-3 before executing guest code. */
7212	pVCpu->hm.s.Event.fPending = false;
7213	}
7214	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7215	return rc;
7216	}
7217	else
7218	{
7219	/*
7220	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7221	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7222	*/
7223	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7224	}
7225	}
7226
7227	/* Validate. */
7228	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7229	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7230	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7231
7232	/* Inject. */
7233	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7234	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7235	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7236	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7237
7238	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7239	&& uVector == X86_XCPT_PF)
7240	{
7241	pMixedCtx->cr2 = GCPtrFaultAddress;
7242	}
7243
7244	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7245	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7246
7247	AssertRCReturn(rc, rc);
7248	return rc;
7249	}
7250
7251
7252	/**
7253	* Clears the interrupt-window exiting control in the VMCS and if necessary
7254	* clears the current event in the VMCS as well.
7255	*
7256	* @returns VBox status code.
7257	* @param pVCpu Pointer to the VMCPU.
7258	*
7259	* @remarks Use this function only to clear events that have not yet been
7260	* delivered to the guest but are injected in the VMCS!
7261	* @remarks No-long-jump zone!!!
7262	*/
7263	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7264	{
7265	int rc;
7266	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7267
7268	/* Clear interrupt-window exiting control. */
7269	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7270	{
7271	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7272	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7273	AssertRC(rc);
7274	}
7275
7276	if (!pVCpu->hm.s.Event.fPending)
7277	return;
7278
7279	#ifdef VBOX_STRICT
7280	uint32_t u32EntryInfo;
7281	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7282	AssertRC(rc);
7283	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7284	#endif
7285
7286	/* Clear the entry-interruption field (including the valid bit). */
7287	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7288	AssertRC(rc);
7289
7290	/* Clear the pending debug exception field. */
7291	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7292	AssertRC(rc);
7293	}
7294
7295
7296	/**
7297	* Enters the VT-x session.
7298	*
7299	* @returns VBox status code.
7300	* @param pVM Pointer to the VM.
7301	* @param pVCpu Pointer to the VMCPU.
7302	* @param pCpu Pointer to the CPU info struct.
7303	*/
7304	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7305	{
7306	AssertPtr(pVM);
7307	AssertPtr(pVCpu);
7308	Assert(pVM->hm.s.vmx.fSupported);
7309	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7310	NOREF(pCpu);
7311
7312	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7313	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7314
7315	#ifdef VBOX_STRICT
7316	/* Make sure we're in VMX root mode. */
7317	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7318	if (!(u32HostCR4 & X86_CR4_VMXE))
7319	{
7320	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7321	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7322	}
7323	#endif
7324
7325	/*
7326	* Load the VCPU's VMCS as the current (and active) one.
7327	*/
7328	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7329	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7330	if (RT_FAILURE(rc))
7331	return rc;
7332
7333	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7334	pVCpu->hm.s.fLeaveDone = false;
7335	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7336
7337	return VINF_SUCCESS;
7338	}
7339
7340
7341	/**
7342	* The thread-context callback (only on platforms which support it).
7343	*
7344	* @param enmEvent The thread-context event.
7345	* @param pVCpu Pointer to the VMCPU.
7346	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7347	* @thread EMT(pVCpu)
7348	*/
7349	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7350	{
7351	switch (enmEvent)
7352	{
7353	case RTTHREADCTXEVENT_PREEMPTING:
7354	{
7355	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7356	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7357	VMCPU_ASSERT_EMT(pVCpu);
7358
7359	PVM pVM = pVCpu->CTX_SUFF(pVM);
7360	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7361
7362	/* No longjmps (logger flushes, locks) in this fragile context. */
7363	VMMRZCallRing3Disable(pVCpu);
7364	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7365
7366	/*
7367	* Restore host-state (FPU, debug etc.)
7368	*/
7369	if (!pVCpu->hm.s.fLeaveDone)
7370	{
7371	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7372	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7373	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7374	pVCpu->hm.s.fLeaveDone = true;
7375	}
7376
7377	/* Leave HM context, takes care of local init (term). */
7378	int rc = HMR0LeaveCpu(pVCpu);
7379	AssertRC(rc); NOREF(rc);
7380
7381	/* Restore longjmp state. */
7382	VMMRZCallRing3Enable(pVCpu);
7383	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7384	break;
7385	}
7386
7387	case RTTHREADCTXEVENT_RESUMED:
7388	{
7389	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7390	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7391	VMCPU_ASSERT_EMT(pVCpu);
7392
7393	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7394	VMMRZCallRing3Disable(pVCpu);
7395	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7396
7397	/* Initialize the bare minimum state required for HM. This takes care of
7398	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7399	int rc = HMR0EnterCpu(pVCpu);
7400	AssertRC(rc);
7401	Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7402
7403	/* Load the active VMCS as the current one. */
7404	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7405	{
7406	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7407	AssertRC(rc); NOREF(rc);
7408	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7409	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7410	}
7411	pVCpu->hm.s.fLeaveDone = false;
7412
7413	/* Restore longjmp state. */
7414	VMMRZCallRing3Enable(pVCpu);
7415	break;
7416	}
7417
7418	default:
7419	break;
7420	}
7421	}
7422
7423
7424	/**
7425	* Saves the host state in the VMCS host-state.
7426	* Sets up the VM-exit MSR-load area.
7427	*
7428	* The CPU state will be loaded from these fields on every successful VM-exit.
7429	*
7430	* @returns VBox status code.
7431	* @param pVM Pointer to the VM.
7432	* @param pVCpu Pointer to the VMCPU.
7433	*
7434	* @remarks No-long-jump zone!!!
7435	*/
7436	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7437	{
7438	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7439
7440	if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7441	return VINF_SUCCESS;
7442
7443	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7444	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7445
7446	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7447	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7448
7449	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7451
7452	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7453	return rc;
7454	}
7455
7456
7457	/**
7458	* Saves the host state in the VMCS host-state.
7459	*
7460	* @returns VBox status code.
7461	* @param pVM Pointer to the VM.
7462	* @param pVCpu Pointer to the VMCPU.
7463	*
7464	* @remarks No-long-jump zone!!!
7465	*/
7466	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7467	{
7468	AssertPtr(pVM);
7469	AssertPtr(pVCpu);
7470
7471	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7472
7473	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7474	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7475	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7476	return hmR0VmxSaveHostState(pVM, pVCpu);
7477	}
7478
7479
7480	/**
7481	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7482	* loaded from these fields on every successful VM-entry.
7483	*
7484	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7485	* Sets up the VM-entry controls.
7486	* Sets up the appropriate VMX non-root function to execute guest code based on
7487	* the guest CPU mode.
7488	*
7489	* @returns VBox status code.
7490	* @param pVM Pointer to the VM.
7491	* @param pVCpu Pointer to the VMCPU.
7492	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7493	* out-of-sync. Make sure to update the required fields
7494	* before using them.
7495	*
7496	* @remarks No-long-jump zone!!!
7497	*/
7498	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7499	{
7500	AssertPtr(pVM);
7501	AssertPtr(pVCpu);
7502	AssertPtr(pMixedCtx);
7503	HMVMX_ASSERT_PREEMPT_SAFE();
7504
7505	#ifdef LOG_ENABLED
7506	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7507	* probably not initialized yet? Anyway this will do for now.
7508	*
7509	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7510	* interface and disable ring-3 calls when thread-context hooks are not
7511	* available. */
7512	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7513	VMMR0LogFlushDisable(pVCpu);
7514	#endif
7515
7516	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7517
7518	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7519
7520	/* Determine real-on-v86 mode. */
7521	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7522	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7523	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7524	{
7525	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7526	}
7527
7528	/*
7529	* Load the guest-state into the VMCS.
7530	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7531	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7532	*/
7533	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7534	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7535
7536	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7537	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7538	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7539
7540	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7541	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7542	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7543
7544	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7545	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7546
7547	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7548	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7549
7550	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7551	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7552	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7553
7554	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7555	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7556
7557	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7558	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7559
7560	/*
7561	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7562	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7563	*/
7564	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7565	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7566
7567	/* Clear any unused and reserved bits. */
7568	VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7569
7570	#ifdef LOG_ENABLED
7571	/* Only reenable log-flushing if the caller has it enabled. */
7572	if (!fCallerDisabledLogFlush)
7573	VMMR0LogFlushEnable(pVCpu);
7574	#endif
7575
7576	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7577	return rc;
7578	}
7579
7580
7581	/**
7582	* Loads the state shared between the host and guest into the VMCS.
7583	*
7584	* @param pVM Pointer to the VM.
7585	* @param pVCpu Pointer to the VMCPU.
7586	* @param pCtx Pointer to the guest-CPU context.
7587	*
7588	* @remarks No-long-jump zone!!!
7589	*/
7590	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7591	{
7592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7593	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7594
7595	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7596	{
7597	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7598	AssertRC(rc);
7599	}
7600
7601	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7602	{
7603	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7604	AssertRC(rc);
7605
7606	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7607	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7608	{
7609	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7610	AssertRC(rc);
7611	}
7612	}
7613
7614	AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7615	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7616	}
7617
7618
7619	/**
7620	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7621	*
7622	* @param pVM Pointer to the VM.
7623	* @param pVCpu Pointer to the VMCPU.
7624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7625	* out-of-sync. Make sure to update the required fields
7626	* before using them.
7627	*/
7628	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7629	{
7630	HMVMX_ASSERT_PREEMPT_SAFE();
7631
7632	Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7633	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7634	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7635	#endif
7636
7637	if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7638	{
7639	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7640	AssertRC(rc);
7641	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7642	}
7643	else if (VMCPU_HMCF_VALUE(pVCpu))
7644	{
7645	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7646	AssertRC(rc);
7647	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7648	}
7649
7650	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7651	AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7652	\|\| VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7653	("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7654
7655	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7656	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7657	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7658	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7659	#endif
7660	}
7661
7662
7663	/**
7664	* Does the preparations before executing guest code in VT-x.
7665	*
7666	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7667	* recompiler. We must be cautious what we do here regarding committing
7668	* guest-state information into the VMCS assuming we assuredly execute the
7669	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7670	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7671	* so that the recompiler can (and should) use them when it resumes guest
7672	* execution. Otherwise such operations must be done when we can no longer
7673	* exit to ring-3.
7674	*
7675	* @returns Strict VBox status code.
7676	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7677	* have been disabled.
7678	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7679	* double-fault into the guest.
7680	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7681	*
7682	* @param pVM Pointer to the VM.
7683	* @param pVCpu Pointer to the VMCPU.
7684	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7685	* out-of-sync. Make sure to update the required fields
7686	* before using them.
7687	* @param pVmxTransient Pointer to the VMX transient structure.
7688	*
7689	* @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7690	* interrupts will be disabled.
7691	*/
7692	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7693	{
7694	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7695
7696	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7697	PGMRZDynMapFlushAutoSet(pVCpu);
7698	#endif
7699
7700	/* Check force flag actions that might require us to go back to ring-3. */
7701	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7702	if (rc != VINF_SUCCESS)
7703	return rc;
7704
7705	#ifndef IEM_VERIFICATION_MODE_FULL
7706	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7707	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7708	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7709	{
7710	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7711	RTGCPHYS GCPhysApicBase;
7712	GCPhysApicBase = pMixedCtx->msrApicBase;
7713	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7714
7715	/* Unalias any existing mapping. */
7716	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7717	AssertRCReturn(rc, rc);
7718
7719	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7720	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7721	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7722	AssertRCReturn(rc, rc);
7723
7724	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7725	}
7726	#endif /* !IEM_VERIFICATION_MODE_FULL */
7727
7728	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7729	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7730
7731	/*
7732	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7733	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7734	*/
7735	if (TRPMHasTrap(pVCpu))
7736	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7737	else if (!pVCpu->hm.s.Event.fPending)
7738	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7739
7740	/*
7741	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7742	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7743	*/
7744	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7745	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7746	{
7747	Assert(rc == VINF_EM_RESET);
7748	return rc;
7749	}
7750
7751	/*
7752	* No longjmps to ring-3 from this point on!!!
7753	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7754	* This also disables flushing of the R0-logger instance (if any).
7755	*/
7756	VMMRZCallRing3Disable(pVCpu);
7757
7758	/*
7759	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7760	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7761	*
7762	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7763	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7764	*
7765	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7766	* executing guest code.
7767	*/
7768	pVmxTransient->uEflags = ASMIntDisableFlags();
7769	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7770	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7771	{
7772	hmR0VmxClearEventVmcs(pVCpu);
7773	ASMSetFlags(pVmxTransient->uEflags);
7774	VMMRZCallRing3Enable(pVCpu);
7775	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7776	return VINF_EM_RAW_TO_R3;
7777	}
7778	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7779	{
7780	hmR0VmxClearEventVmcs(pVCpu);
7781	ASMSetFlags(pVmxTransient->uEflags);
7782	VMMRZCallRing3Enable(pVCpu);
7783	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7784	return VINF_EM_RAW_INTERRUPT;
7785	}
7786
7787	/* We've injected any pending events. This is really the point of no return (to ring-3). */
7788	pVCpu->hm.s.Event.fPending = false;
7789
7790	return VINF_SUCCESS;
7791	}
7792
7793
7794	/**
7795	* Prepares to run guest code in VT-x and we've committed to doing so. This
7796	* means there is no backing out to ring-3 or anywhere else at this
7797	* point.
7798	*
7799	* @param pVM Pointer to the VM.
7800	* @param pVCpu Pointer to the VMCPU.
7801	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7802	* out-of-sync. Make sure to update the required fields
7803	* before using them.
7804	* @param pVmxTransient Pointer to the VMX transient structure.
7805	*
7806	* @remarks Called with preemption disabled.
7807	* @remarks No-long-jump zone!!!
7808	*/
7809	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7810	{
7811	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7812	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7813	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7814
7815	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7816	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7817
7818	/*
7819	* If we are injecting events to a real-on-v86 mode guest, we may have to update
7820	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7821	* Reload only the necessary state, the assertion will catch if other parts of the code
7822	* change.
7823	*/
7824	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7825	{
7826	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7827	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7828	}
7829
7830	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7831	if (!CPUMIsGuestFPUStateActive(pVCpu))
7832	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7833	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7834	#endif
7835
7836	if ( pVCpu->hm.s.fUseGuestFpu
7837	&& !CPUMIsGuestFPUStateActive(pVCpu))
7838	{
7839	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7840	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
7841	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7842	}
7843
7844	/*
7845	* Load the host state bits as we may've been preempted (only happens when
7846	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7847	*/
7848	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7849	{
7850	/* This ASSUMES that pfnStartVM has been set up already. */
7851	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7852	AssertRC(rc);
7853	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7854	}
7855	Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7856
7857	/*
7858	* Load the state shared between host and guest (FPU, debug).
7859	*/
7860	if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7861	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7862	AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7863
7864	/* Store status of the shared guest-host state at the time of VM-entry. */
7865	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7866	if (CPUMIsGuestInLongModeEx(pMixedCtx))
7867	{
7868	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7869	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7870	}
7871	else
7872	#endif
7873	{
7874	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7875	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7876	}
7877	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7878
7879	/*
7880	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7881	*/
7882	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7883	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7884
7885	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7886	RTCPUID idCurrentCpu = pCpu->idCpu;
7887	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7888	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
7889	{
7890	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7891	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7892	}
7893
7894	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7895	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7896	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7897	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7898
7899	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7900
7901	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7902	to start executing. */
7903
7904	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7905	/*
7906	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7907	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7908	*/
7909	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7910	{
7911	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7912	uint64_t u64HostTscAux = 0;
7913	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7914	AssertRC(rc2);
7915	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7916	}
7917	#endif
7918	}
7919
7920
7921	/**
7922	* Performs some essential restoration of state after running guest code in
7923	* VT-x.
7924	*
7925	* @param pVM Pointer to the VM.
7926	* @param pVCpu Pointer to the VMCPU.
7927	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7928	* out-of-sync. Make sure to update the required fields
7929	* before using them.
7930	* @param pVmxTransient Pointer to the VMX transient structure.
7931	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7932	*
7933	* @remarks Called with interrupts disabled, and returns with interrups enabled!
7934	*
7935	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
7936	* unconditionally when it is safe to do so.
7937	*/
7938	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7939	{
7940	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7941
7942	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7943	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7944	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7945	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7946	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7947
7948	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7949	{
7950	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7951	/* Restore host's TSC_AUX. */
7952	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7953	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7954	#endif
7955	/** @todo Find a way to fix hardcoding a guestimate. */
7956	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7957	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7958	}
7959
7960	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7961	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7962	Assert(!(ASMGetFlags() & X86_EFL_IF));
7963	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7964
7965	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7966	if (CPUMIsGuestFPUStateActive(pVCpu))
7967	{
7968	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7969	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7970	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7971	}
7972	#endif
7973
7974	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7975	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7976	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7977	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7978
7979	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7980	uint32_t uExitReason;
7981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7982	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
7983	AssertRC(rc);
7984	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7985	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
7986
7987	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
7988	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7989	{
7990	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7991	pVmxTransient->fVMEntryFailed));
7992	return;
7993	}
7994
7995	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7996	{
7997	/* Update the guest interruptibility-state from the VMCS. */
7998	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7999	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8000	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8001	AssertRC(rc);
8002	#endif
8003	/*
8004	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8005	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8006	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8007	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8008	*/
8009	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8010	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8011	{
8012	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8013	AssertRC(rc);
8014	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8015	}
8016	}
8017	}
8018
8019
8020
8021	/**
8022	* Runs the guest code using VT-x the normal way.
8023	*
8024	* @returns VBox status code.
8025	* @param pVM Pointer to the VM.
8026	* @param pVCpu Pointer to the VMCPU.
8027	* @param pCtx Pointer to the guest-CPU context.
8028	*
8029	* @note Mostly the same as hmR0VmxRunGuestCodeStep.
8030	* @remarks Called with preemption disabled.
8031	*/
8032	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8033	{
8034	VMXTRANSIENT VmxTransient;
8035	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8036	int rc = VERR_INTERNAL_ERROR_5;
8037	uint32_t cLoops = 0;
8038
8039	for (;; cLoops++)
8040	{
8041	Assert(!HMR0SuspendPending());
8042	HMVMX_ASSERT_CPU_SAFE();
8043
8044	/* Preparatory work for running guest code, this may force us to return
8045	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8046	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8047	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8048	if (rc != VINF_SUCCESS)
8049	break;
8050
8051	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8052	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8053	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8054
8055	/* Restore any residual host-state and save any bits shared between host
8056	and guest into the guest-CPU state. Re-enables interrupts! */
8057	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8058
8059	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8060	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8061	{
8062	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8063	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8064	return rc;
8065	}
8066
8067	/* Handle the VM-exit. */
8068	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8070	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8071	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8072	HMVMX_START_EXIT_DISPATCH_PROF();
8073	#ifdef HMVMX_USE_FUNCTION_TABLE
8074	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8075	#else
8076	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8077	#endif
8078	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8079	if (rc != VINF_SUCCESS)
8080	break;
8081	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8082	{
8083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8084	rc = VINF_EM_RAW_INTERRUPT;
8085	break;
8086	}
8087	}
8088
8089	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8090	return rc;
8091	}
8092
8093
8094	/**
8095	* Single steps guest code using VT-x.
8096	*
8097	* @returns VBox status code.
8098	* @param pVM Pointer to the VM.
8099	* @param pVCpu Pointer to the VMCPU.
8100	* @param pCtx Pointer to the guest-CPU context.
8101	*
8102	* @note Mostly the same as hmR0VmxRunGuestCodeNormal.
8103	* @remarks Called with preemption disabled.
8104	*/
8105	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8106	{
8107	VMXTRANSIENT VmxTransient;
8108	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8109	int rc = VERR_INTERNAL_ERROR_5;
8110	uint32_t cLoops = 0;
8111	uint16_t uCsStart = pCtx->cs.Sel;
8112	uint64_t uRipStart = pCtx->rip;
8113
8114	for (;; cLoops++)
8115	{
8116	Assert(!HMR0SuspendPending());
8117	HMVMX_ASSERT_CPU_SAFE();
8118
8119	/* Preparatory work for running guest code, this may force us to return
8120	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8121	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8122	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8123	if (rc != VINF_SUCCESS)
8124	break;
8125
8126	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8127	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8128	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8129
8130	/* Restore any residual host-state and save any bits shared between host
8131	and guest into the guest-CPU state. Re-enables interrupts! */
8132	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8133
8134	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8135	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8136	{
8137	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8138	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8139	return rc;
8140	}
8141
8142	/* Handle the VM-exit. */
8143	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8145	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8146	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8147	HMVMX_START_EXIT_DISPATCH_PROF();
8148	#ifdef HMVMX_USE_FUNCTION_TABLE
8149	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8150	#else
8151	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8152	#endif
8153	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8154	if (rc != VINF_SUCCESS)
8155	break;
8156	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8157	{
8158	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8159	rc = VINF_EM_RAW_INTERRUPT;
8160	break;
8161	}
8162
8163	/*
8164	* Did the RIP change, if so, consider it a single step.
8165	* Otherwise, make sure one of the TFs gets set.
8166	*/
8167	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8168	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8169	AssertRCReturn(rc2, rc2);
8170	if ( pCtx->rip != uRipStart
8171	\|\| pCtx->cs.Sel != uCsStart)
8172	{
8173	rc = VINF_EM_DBG_STEPPED;
8174	break;
8175	}
8176	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8177	}
8178
8179	/*
8180	* Clear the X86_EFL_TF if necessary.
8181	*/
8182	if (pVCpu->hm.s.fClearTrapFlag)
8183	{
8184	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8185	AssertRCReturn(rc2, rc2);
8186	pVCpu->hm.s.fClearTrapFlag = false;
8187	pCtx->eflags.Bits.u1TF = 0;
8188	}
8189	/** @todo there seems to be issues with the resume flag when the monitor trap
8190	* flag is pending without being used. Seen early in bios init when
8191	* accessing APIC page in prot mode. */
8192
8193	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8194	return rc;
8195	}
8196
8197
8198	/**
8199	* Runs the guest code using VT-x.
8200	*
8201	* @returns VBox status code.
8202	* @param pVM Pointer to the VM.
8203	* @param pVCpu Pointer to the VMCPU.
8204	* @param pCtx Pointer to the guest-CPU context.
8205	*
8206	* @remarks Called with preemption disabled.
8207	*/
8208	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8209	{
8210	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8211	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
8212	HMVMX_ASSERT_PREEMPT_SAFE();
8213
8214	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8215
8216	int rc;
8217	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8218	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8219	else
8220	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8221
8222	if (rc == VERR_EM_INTERPRETER)
8223	rc = VINF_EM_RAW_EMULATE_INSTR;
8224	else if (rc == VINF_EM_RESET)
8225	rc = VINF_EM_TRIPLE_FAULT;
8226
8227	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8228	if (RT_FAILURE(rc2))
8229	{
8230	pVCpu->hm.s.u32HMError = rc;
8231	rc = rc2;
8232	}
8233	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8234	return rc;
8235	}
8236
8237
8238	#ifndef HMVMX_USE_FUNCTION_TABLE
8239	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8240	{
8241	int rc;
8242	switch (rcReason)
8243	{
8244	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
8245	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
8246	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
8247	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
8248	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
8249	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
8250	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8251	case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8252	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8253	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8254	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8255	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8256	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8257	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8258	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8259	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8260	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8261	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8262	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8263	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8264	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8265	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8266	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8267	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8268	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8269	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8270	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8271	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8272	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8273	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8274	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8275	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8276	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8277
8278	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8279	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8280	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8281	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8282	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8283	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8284	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8285	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8286	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8287
8288	case VMX_EXIT_VMCALL:
8289	case VMX_EXIT_VMCLEAR:
8290	case VMX_EXIT_VMLAUNCH:
8291	case VMX_EXIT_VMPTRLD:
8292	case VMX_EXIT_VMPTRST:
8293	case VMX_EXIT_VMREAD:
8294	case VMX_EXIT_VMRESUME:
8295	case VMX_EXIT_VMWRITE:
8296	case VMX_EXIT_VMXOFF:
8297	case VMX_EXIT_VMXON:
8298	case VMX_EXIT_INVEPT:
8299	case VMX_EXIT_INVVPID:
8300	case VMX_EXIT_VMFUNC:
8301	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8302	break;
8303	default:
8304	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8305	break;
8306	}
8307	return rc;
8308	}
8309	#endif
8310
8311	#ifdef DEBUG
8312	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8313	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8314	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8315
8316	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8317	do \
8318	{ \
8319	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8320	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8321	} while (0)
8322
8323	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8324	do { \
8325	AssertPtr(pVCpu); \
8326	AssertPtr(pMixedCtx); \
8327	AssertPtr(pVmxTransient); \
8328	Assert(pVmxTransient->fVMEntryFailed == false); \
8329	Assert(ASMIntAreEnabled()); \
8330	HMVMX_ASSERT_PREEMPT_SAFE(); \
8331	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8332	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8333	HMVMX_ASSERT_PREEMPT_SAFE(); \
8334	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8335	HMVMX_ASSERT_PREEMPT_CPUID(); \
8336	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8337	} while (0)
8338
8339	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8340	do { \
8341	Log4Func(("\n")); \
8342	} while(0)
8343	#else /* Release builds */
8344	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8345	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8346	#endif
8347
8348
8349	/**
8350	* Advances the guest RIP after reading it from the VMCS.
8351	*
8352	* @returns VBox status code.
8353	* @param pVCpu Pointer to the VMCPU.
8354	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8355	* out-of-sync. Make sure to update the required fields
8356	* before using them.
8357	* @param pVmxTransient Pointer to the VMX transient structure.
8358	*
8359	* @remarks No-long-jump zone!!!
8360	*/
8361	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8362	{
8363	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8364	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8365	AssertRCReturn(rc, rc);
8366
8367	pMixedCtx->rip += pVmxTransient->cbInstr;
8368	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8369	return rc;
8370	}
8371
8372
8373	/**
8374	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8375	* and update error record fields accordingly.
8376	*
8377	* @return VMX_IGS_* return codes.
8378	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8379	* wrong with the guest state.
8380	*
8381	* @param pVM Pointer to the VM.
8382	* @param pVCpu Pointer to the VMCPU.
8383	* @param pCtx Pointer to the guest-CPU state.
8384	*/
8385	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8386	{
8387	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8388	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8389	uError = (err); \
8390	break; \
8391	} else do {} while (0)
8392	/* Duplicate of IEM_IS_CANONICAL(). */
8393	#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8394
8395	int rc;
8396	uint32_t uError = VMX_IGS_ERROR;
8397	uint32_t u32Val;
8398	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8399
8400	do
8401	{
8402	/*
8403	* CR0.
8404	*/
8405	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8406	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8407	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8408	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8409	if (fUnrestrictedGuest)
8410	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8411
8412	uint32_t u32GuestCR0;
8413	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8414	AssertRCBreak(rc);
8415	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8416	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8417	if ( !fUnrestrictedGuest
8418	&& (u32GuestCR0 & X86_CR0_PG)
8419	&& !(u32GuestCR0 & X86_CR0_PE))
8420	{
8421	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8422	}
8423
8424	/*
8425	* CR4.
8426	*/
8427	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8428	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8429
8430	uint32_t u32GuestCR4;
8431	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8432	AssertRCBreak(rc);
8433	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8434	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8435
8436	/*
8437	* IA32_DEBUGCTL MSR.
8438	*/
8439	uint64_t u64Val;
8440	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8441	AssertRCBreak(rc);
8442	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8443	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8444	{
8445	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8446	}
8447	uint64_t u64DebugCtlMsr = u64Val;
8448
8449	#ifdef VBOX_STRICT
8450	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8451	AssertRCBreak(rc);
8452	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8453	#endif
8454	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8455
8456	/*
8457	* RIP and RFLAGS.
8458	*/
8459	uint32_t u32Eflags;
8460	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8461	if (HMVMX_IS_64BIT_HOST_MODE())
8462	{
8463	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8464	AssertRCBreak(rc);
8465	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8466	if ( !fLongModeGuest
8467	\|\| !pCtx->cs.Attr.n.u1Long)
8468	{
8469	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8470	}
8471	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8472	* must be identical if the "IA32e mode guest" VM-entry control is 1
8473	* and CS.L is 1. No check applies if the CPU supports 64
8474	* linear-address bits. */
8475
8476	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8477	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8478	AssertRCBreak(rc);
8479	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8480	VMX_IGS_RFLAGS_RESERVED);
8481	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8482	u32Eflags = u64Val;
8483	}
8484	else
8485	#endif
8486	{
8487	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8488	AssertRCBreak(rc);
8489	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8490	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8491	}
8492
8493	if ( fLongModeGuest
8494	\|\| ( fUnrestrictedGuest
8495	&& !(u32GuestCR0 & X86_CR0_PE)))
8496	{
8497	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8498	}
8499
8500	uint32_t u32EntryInfo;
8501	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8502	AssertRCBreak(rc);
8503	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8504	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8505	{
8506	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8507	}
8508
8509	/*
8510	* 64-bit checks.
8511	*/
8512	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8513	if (HMVMX_IS_64BIT_HOST_MODE())
8514	{
8515	if ( fLongModeGuest
8516	&& !fUnrestrictedGuest)
8517	{
8518	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8519	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8520	}
8521
8522	if ( !fLongModeGuest
8523	&& (u32GuestCR4 & X86_CR4_PCIDE))
8524	{
8525	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8526	}
8527
8528	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8529	* 51:32 beyond the processor's physical-address width are 0. */
8530
8531	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8532	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8533	{
8534	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8535	}
8536
8537	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8538	AssertRCBreak(rc);
8539	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8540
8541	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8542	AssertRCBreak(rc);
8543	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8544	}
8545	#endif
8546
8547	/*
8548	* PERF_GLOBAL MSR.
8549	*/
8550	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8551	{
8552	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8553	AssertRCBreak(rc);
8554	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8555	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8556	}
8557
8558	/*
8559	* PAT MSR.
8560	*/
8561	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8562	{
8563	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8564	AssertRCBreak(rc);
8565	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8566	for (unsigned i = 0; i < 8; i++)
8567	{
8568	uint8_t u8Val = (u64Val & 0x7);
8569	if ( u8Val != 0 /* UC */
8570	\|\| u8Val != 1 /* WC */
8571	\|\| u8Val != 4 /* WT */
8572	\|\| u8Val != 5 /* WP */
8573	\|\| u8Val != 6 /* WB */
8574	\|\| u8Val != 7 /* UC- */)
8575	{
8576	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8577	}
8578	u64Val >>= 3;
8579	}
8580	}
8581
8582	/*
8583	* EFER MSR.
8584	*/
8585	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8586	{
8587	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8588	AssertRCBreak(rc);
8589	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8590	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8591	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8592	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8593	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8594	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8595	}
8596
8597	/*
8598	* Segment registers.
8599	*/
8600	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8601	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8602	if (!(u32Eflags & X86_EFL_VM))
8603	{
8604	/* CS */
8605	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8606	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8607	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8608	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8609	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8610	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8611	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8612	/* CS cannot be loaded with NULL in protected mode. */
8613	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8614	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8615	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8616	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8617	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8618	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8619	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8620	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8621	else
8622	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8623
8624	/* SS */
8625	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8626	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8627	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8628	if ( !(pCtx->cr0 & X86_CR0_PE)
8629	\|\| pCtx->cs.Attr.n.u4Type == 3)
8630	{
8631	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8632	}
8633	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8634	{
8635	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8636	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8637	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8638	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8639	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8640	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8641	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8642	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8643	}
8644
8645	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8646	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8647	{
8648	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8649	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8650	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8651	\|\| pCtx->ds.Attr.n.u4Type > 11
8652	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8653	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8654	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8655	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8656	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8657	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8658	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8659	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8660	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8661	}
8662	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8663	{
8664	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8665	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8666	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8667	\|\| pCtx->es.Attr.n.u4Type > 11
8668	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8669	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8670	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8671	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8672	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8673	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8674	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8675	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8676	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8677	}
8678	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8679	{
8680	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8681	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8682	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8683	\|\| pCtx->fs.Attr.n.u4Type > 11
8684	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8685	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8686	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8687	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8688	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8689	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8690	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8691	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8692	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8693	}
8694	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8695	{
8696	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8697	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8698	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8699	\|\| pCtx->gs.Attr.n.u4Type > 11
8700	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8701	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8702	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8703	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8704	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8705	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8706	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8707	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8708	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8709	}
8710	/* 64-bit capable CPUs. */
8711	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8712	if (HMVMX_IS_64BIT_HOST_MODE())
8713	{
8714	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8715	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8716	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8717	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8718	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8719	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8720	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8721	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8722	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8723	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8724	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8725	}
8726	#endif
8727	}
8728	else
8729	{
8730	/* V86 mode checks. */
8731	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8732	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8733	{
8734	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8735	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8736	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8737	}
8738	else
8739	{
8740	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8741	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8742	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8743	}
8744
8745	/* CS */
8746	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8747	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8748	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8749	/* SS */
8750	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8751	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8752	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8753	/* DS */
8754	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8755	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8756	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8757	/* ES */
8758	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8759	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8760	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8761	/* FS */
8762	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8763	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8764	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8765	/* GS */
8766	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8767	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8768	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8769	/* 64-bit capable CPUs. */
8770	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8771	if (HMVMX_IS_64BIT_HOST_MODE())
8772	{
8773	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8774	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8775	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8776	\|\| HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8777	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8778	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8779	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8780	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8781	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8782	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8783	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8784	}
8785	#endif
8786	}
8787
8788	/*
8789	* TR.
8790	*/
8791	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8792	/* 64-bit capable CPUs. */
8793	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8794	if (HMVMX_IS_64BIT_HOST_MODE())
8795	{
8796	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8797	}
8798	#endif
8799	if (fLongModeGuest)
8800	{
8801	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8802	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8803	}
8804	else
8805	{
8806	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8807	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8808	VMX_IGS_TR_ATTR_TYPE_INVALID);
8809	}
8810	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8811	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8812	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8813	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8814	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8815	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8816	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8817	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8818
8819	/*
8820	* GDTR and IDTR.
8821	*/
8822	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8823	if (HMVMX_IS_64BIT_HOST_MODE())
8824	{
8825	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8826	AssertRCBreak(rc);
8827	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8828
8829	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8830	AssertRCBreak(rc);
8831	HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8832	}
8833	#endif
8834
8835	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8836	AssertRCBreak(rc);
8837	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8838
8839	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8840	AssertRCBreak(rc);
8841	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8842
8843	/*
8844	* Guest Non-Register State.
8845	*/
8846	/* Activity State. */
8847	uint32_t u32ActivityState;
8848	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8849	AssertRCBreak(rc);
8850	HMVMX_CHECK_BREAK( !u32ActivityState
8851	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8852	VMX_IGS_ACTIVITY_STATE_INVALID);
8853	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8854	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8855	uint32_t u32IntrState;
8856	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8857	AssertRCBreak(rc);
8858	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8859	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8860	{
8861	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8862	}
8863
8864	/** @todo Activity state and injecting interrupts. Left as a todo since we
8865	* currently don't use activity states but ACTIVE. */
8866
8867	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8868	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8869
8870	/* Guest interruptibility-state. */
8871	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8872	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8873	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8874	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8875	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8876	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8877	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8878	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8879	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8880	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
8881	{
8882	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8883	{
8884	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8885	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8886	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8887	}
8888	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8889	{
8890	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8891	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8892	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8893	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8894	}
8895	}
8896	/** @todo Assumes the processor is not in SMM. */
8897	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8898	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8899	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8900	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8901	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8902	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8903	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8904	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8905	{
8906	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8907	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8908	}
8909
8910	/* Pending debug exceptions. */
8911	if (HMVMX_IS_64BIT_HOST_MODE())
8912	{
8913	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8914	AssertRCBreak(rc);
8915	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8916	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8917	u32Val = u64Val; /* For pending debug exceptions checks below. */
8918	}
8919	else
8920	{
8921	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8922	AssertRCBreak(rc);
8923	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8924	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8925	}
8926
8927	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8928	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8929	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8930	{
8931	if ( (u32Eflags & X86_EFL_TF)
8932	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8933	{
8934	/* Bit 14 is PendingDebug.BS. */
8935	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8936	}
8937	if ( !(u32Eflags & X86_EFL_TF)
8938	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8939	{
8940	/* Bit 14 is PendingDebug.BS. */
8941	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8942	}
8943	}
8944
8945	/* VMCS link pointer. */
8946	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8947	AssertRCBreak(rc);
8948	if (u64Val != UINT64_C(0xffffffffffffffff))
8949	{
8950	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8951	/** @todo Bits beyond the processor's physical-address width MBZ. */
8952	/** @todo 32-bit located in memory referenced by value of this field (as a
8953	* physical address) must contain the processor's VMCS revision ID. */
8954	/** @todo SMM checks. */
8955	}
8956
8957	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8958
8959	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8960	if (uError == VMX_IGS_ERROR)
8961	uError = VMX_IGS_REASON_NOT_FOUND;
8962	} while (0);
8963
8964	pVCpu->hm.s.u32HMError = uError;
8965	return uError;
8966
8967	#undef HMVMX_ERROR_BREAK
8968	#undef HMVMX_CHECK_BREAK
8969	#undef HMVMX_IS_CANONICAL
8970	}
8971
8972	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8973	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8974	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8975
8976	/** @name VM-exit handlers.
8977	* @{
8978	*/
8979
8980	/**
8981	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8982	*/
8983	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8984	{
8985	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8986	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8987	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8988	#if HC_ARCH_BITS == 64
8989	Assert(ASMIntAreEnabled());
8990	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8991	return VINF_SUCCESS;
8992	#endif
8993	return VINF_EM_RAW_INTERRUPT;
8994	}
8995
8996
8997	/**
8998	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8999	*/
9000	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9001	{
9002	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9003	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9004
9005	int rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
9006	AssertRCReturn(rc, rc);
9007
9008	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9009	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9010	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9011	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9012
9013	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9014	{
9015	/*
9016	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9017	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9018	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9019	*
9020	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9021	*/
9022	VMXDispatchHostNmi();
9023	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9024	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9025	return VINF_SUCCESS;
9026	}
9027
9028	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9029	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9030	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9031	{
9032	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9033	return VINF_SUCCESS;
9034	}
9035	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9036	{
9037	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9038	return rc;
9039	}
9040
9041	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9042	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9043	switch (uIntType)
9044	{
9045	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9046	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
9047	/* no break */
9048	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9049	{
9050	switch (uVector)
9051	{
9052	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9053	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9054	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9055	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9056	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9057	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9058	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9059	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9060	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9061	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9062	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9063	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9064	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9065	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9066	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9067	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9068	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9069	#endif
9070	default:
9071	{
9072	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9073	AssertRCReturn(rc, rc);
9074
9075	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9076	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9077	{
9078	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9079	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9080	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9081
9082	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9083	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
9084	AssertRCReturn(rc, rc);
9085	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9086	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9087	0 /* GCPtrFaultAddress */);
9088	AssertRCReturn(rc, rc);
9089	}
9090	else
9091	{
9092	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9093	pVCpu->hm.s.u32HMError = uVector;
9094	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9095	}
9096	break;
9097	}
9098	}
9099	break;
9100	}
9101
9102	default:
9103	{
9104	pVCpu->hm.s.u32HMError = uExitIntInfo;
9105	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9106	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9107	break;
9108	}
9109	}
9110	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9111	return rc;
9112	}
9113
9114
9115	/**
9116	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9117	*/
9118	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9119	{
9120	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9121
9122	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9123	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9124	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9125	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9126	AssertRCReturn(rc, rc);
9127
9128	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9130	return VINF_SUCCESS;
9131	}
9132
9133
9134	/**
9135	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9136	*/
9137	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9138	{
9139	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9140	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9141	HMVMX_RETURN_UNEXPECTED_EXIT();
9142	}
9143
9144
9145	/**
9146	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9147	*/
9148	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9149	{
9150	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9151	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9152	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9153	}
9154
9155
9156	/**
9157	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9158	*/
9159	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9160	{
9161	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9163	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9164	}
9165
9166
9167	/**
9168	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9169	*/
9170	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9171	{
9172	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9173	PVM pVM = pVCpu->CTX_SUFF(pVM);
9174	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9175	if (RT_LIKELY(rc == VINF_SUCCESS))
9176	{
9177	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9178	Assert(pVmxTransient->cbInstr == 2);
9179	}
9180	else
9181	{
9182	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9183	rc = VERR_EM_INTERPRETER;
9184	}
9185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9186	return rc;
9187	}
9188
9189
9190	/**
9191	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9192	*/
9193	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9194	{
9195	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9196	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9197	AssertRCReturn(rc, rc);
9198
9199	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9200	return VINF_EM_RAW_EMULATE_INSTR;
9201
9202	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9203	HMVMX_RETURN_UNEXPECTED_EXIT();
9204	}
9205
9206
9207	/**
9208	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9209	*/
9210	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9211	{
9212	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9213	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9214	AssertRCReturn(rc, rc);
9215
9216	PVM pVM = pVCpu->CTX_SUFF(pVM);
9217	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9218	if (RT_LIKELY(rc == VINF_SUCCESS))
9219	{
9220	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9221	Assert(pVmxTransient->cbInstr == 2);
9222	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9223	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9224	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9225	}
9226	else
9227	{
9228	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9229	rc = VERR_EM_INTERPRETER;
9230	}
9231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9232	return rc;
9233	}
9234
9235
9236	/**
9237	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9238	*/
9239	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9240	{
9241	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9242	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9243	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9244	AssertRCReturn(rc, rc);
9245
9246	PVM pVM = pVCpu->CTX_SUFF(pVM);
9247	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9248	if (RT_LIKELY(rc == VINF_SUCCESS))
9249	{
9250	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9251	Assert(pVmxTransient->cbInstr == 3);
9252	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9253	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9254	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9255	}
9256	else
9257	{
9258	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9259	rc = VERR_EM_INTERPRETER;
9260	}
9261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9262	return rc;
9263	}
9264
9265
9266	/**
9267	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9268	*/
9269	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9270	{
9271	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9272	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9273	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9274	AssertRCReturn(rc, rc);
9275
9276	PVM pVM = pVCpu->CTX_SUFF(pVM);
9277	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9278	if (RT_LIKELY(rc == VINF_SUCCESS))
9279	{
9280	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9281	Assert(pVmxTransient->cbInstr == 2);
9282	}
9283	else
9284	{
9285	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9286	rc = VERR_EM_INTERPRETER;
9287	}
9288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9289	return rc;
9290	}
9291
9292
9293	/**
9294	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9295	*/
9296	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9297	{
9298	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9299	PVM pVM = pVCpu->CTX_SUFF(pVM);
9300	Assert(!pVM->hm.s.fNestedPaging);
9301
9302	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9303	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9304	AssertRCReturn(rc, rc);
9305
9306	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9307	rc = VBOXSTRICTRC_VAL(rc2);
9308	if (RT_LIKELY(rc == VINF_SUCCESS))
9309	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9310	else
9311	{
9312	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9313	pVmxTransient->uExitQualification, rc));
9314	}
9315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9316	return rc;
9317	}
9318
9319
9320	/**
9321	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9322	*/
9323	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9324	{
9325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9326	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9327	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9328	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9329	AssertRCReturn(rc, rc);
9330
9331	PVM pVM = pVCpu->CTX_SUFF(pVM);
9332	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9333	if (RT_LIKELY(rc == VINF_SUCCESS))
9334	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9335	else
9336	{
9337	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9338	rc = VERR_EM_INTERPRETER;
9339	}
9340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9341	return rc;
9342	}
9343
9344
9345	/**
9346	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9347	*/
9348	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9349	{
9350	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9351	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9352	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9353	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9354	AssertRCReturn(rc, rc);
9355
9356	PVM pVM = pVCpu->CTX_SUFF(pVM);
9357	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9358	rc = VBOXSTRICTRC_VAL(rc2);
9359	if (RT_LIKELY( rc == VINF_SUCCESS
9360	\|\| rc == VINF_EM_HALT))
9361	{
9362	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9363	AssertRCReturn(rc3, rc3);
9364
9365	if ( rc == VINF_EM_HALT
9366	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9367	{
9368	rc = VINF_SUCCESS;
9369	}
9370	}
9371	else
9372	{
9373	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9374	rc = VERR_EM_INTERPRETER;
9375	}
9376	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9377	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9378	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9379	return rc;
9380	}
9381
9382
9383	/**
9384	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9385	*/
9386	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9387	{
9388	/*
9389	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9390	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9391	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9392	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9393	*/
9394	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9395	HMVMX_RETURN_UNEXPECTED_EXIT();
9396	}
9397
9398
9399	/**
9400	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9401	*/
9402	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9403	{
9404	/*
9405	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9406	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9407	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9408	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9409	*/
9410	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9411	HMVMX_RETURN_UNEXPECTED_EXIT();
9412	}
9413
9414
9415	/**
9416	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9417	*/
9418	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9419	{
9420	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9421	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9422	HMVMX_RETURN_UNEXPECTED_EXIT();
9423	}
9424
9425
9426	/**
9427	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9428	*/
9429	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9430	{
9431	/*
9432	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9433	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9434	* See Intel spec. 25.3 "Other Causes of VM-exits".
9435	*/
9436	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9437	HMVMX_RETURN_UNEXPECTED_EXIT();
9438	}
9439
9440
9441	/**
9442	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9443	* VM-exit.
9444	*/
9445	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9446	{
9447	/*
9448	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9449	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9450	*
9451	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9452	* See Intel spec. "23.8 Restrictions on VMX operation".
9453	*/
9454	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9455	return VINF_SUCCESS;
9456	}
9457
9458
9459	/**
9460	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9461	* VM-exit.
9462	*/
9463	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9464	{
9465	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9466	return VINF_EM_RESET;
9467	}
9468
9469
9470	/**
9471	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9472	*/
9473	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9474	{
9475	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9476	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9477	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9478	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9479	AssertRCReturn(rc, rc);
9480
9481	pMixedCtx->rip++;
9482	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9483	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9484	rc = VINF_SUCCESS;
9485	else
9486	rc = VINF_EM_HALT;
9487
9488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9489	return rc;
9490	}
9491
9492
9493	/**
9494	* VM-exit handler for instructions that result in a #UD exception delivered to
9495	* the guest.
9496	*/
9497	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9498	{
9499	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9500	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9501	return VINF_SUCCESS;
9502	}
9503
9504
9505	/**
9506	* VM-exit handler for expiry of the VMX preemption timer.
9507	*/
9508	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9509	{
9510	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9511
9512	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9513	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9514
9515	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9516	PVM pVM = pVCpu->CTX_SUFF(pVM);
9517	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9519	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9520	}
9521
9522
9523	/**
9524	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9525	*/
9526	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9527	{
9528	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9529
9530	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9531	/** @todo check if XSETBV is supported by the recompiler. */
9532	return VERR_EM_INTERPRETER;
9533	}
9534
9535
9536	/**
9537	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9538	*/
9539	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9540	{
9541	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9542
9543	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9544	/** @todo implement EMInterpretInvpcid() */
9545	return VERR_EM_INTERPRETER;
9546	}
9547
9548
9549	/**
9550	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9551	* Error VM-exit.
9552	*/
9553	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9554	{
9555	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9556	AssertRCReturn(rc, rc);
9557
9558	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9559	NOREF(uInvalidReason);
9560
9561	#ifdef VBOX_STRICT
9562	uint32_t uIntrState;
9563	HMVMXHCUINTREG uHCReg;
9564	uint64_t u64Val;
9565	uint32_t u32Val;
9566
9567	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9568	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9569	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9570	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9571	AssertRCReturn(rc, rc);
9572
9573	Log4(("uInvalidReason %u\n", uInvalidReason));
9574	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9575	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9576	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9577	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9578
9579	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9580	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9581	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9582	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9583	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9584	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9585	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9586	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9587	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9588	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9589	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9590	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9591	#endif
9592
9593	PVM pVM = pVCpu->CTX_SUFF(pVM);
9594	HMDumpRegs(pVM, pVCpu, pMixedCtx);
9595
9596	return VERR_VMX_INVALID_GUEST_STATE;
9597	}
9598
9599
9600	/**
9601	* VM-exit handler for VM-entry failure due to an MSR-load
9602	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9603	*/
9604	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9605	{
9606	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9607	HMVMX_RETURN_UNEXPECTED_EXIT();
9608	}
9609
9610
9611	/**
9612	* VM-exit handler for VM-entry failure due to a machine-check event
9613	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9614	*/
9615	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9616	{
9617	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9618	HMVMX_RETURN_UNEXPECTED_EXIT();
9619	}
9620
9621
9622	/**
9623	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9624	* theory.
9625	*/
9626	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9627	{
9628	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9629	return VERR_VMX_UNDEFINED_EXIT_CODE;
9630	}
9631
9632
9633	/**
9634	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9635	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9636	* Conditional VM-exit.
9637	*/
9638	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9639	{
9640	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9641
9642	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9644	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9645	return VERR_EM_INTERPRETER;
9646	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9647	HMVMX_RETURN_UNEXPECTED_EXIT();
9648	}
9649
9650
9651	/**
9652	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9653	*/
9654	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9655	{
9656	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9657
9658	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9660	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9661	return VERR_EM_INTERPRETER;
9662	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9663	HMVMX_RETURN_UNEXPECTED_EXIT();
9664	}
9665
9666
9667	/**
9668	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9669	*/
9670	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9671	{
9672	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9673
9674	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9675	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9676	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9677	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9678	AssertRCReturn(rc, rc);
9679	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9680
9681	PVM pVM = pVCpu->CTX_SUFF(pVM);
9682	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9683	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9684	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9686
9687	if (RT_LIKELY(rc == VINF_SUCCESS))
9688	{
9689	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9690	Assert(pVmxTransient->cbInstr == 2);
9691	}
9692	return rc;
9693	}
9694
9695
9696	/**
9697	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9698	*/
9699	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9700	{
9701	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9702	PVM pVM = pVCpu->CTX_SUFF(pVM);
9703	int rc = VINF_SUCCESS;
9704
9705	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9706	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9707	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9708	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9709	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9710	AssertRCReturn(rc, rc);
9711	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9712
9713	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9714	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9716
9717	if (RT_LIKELY(rc == VINF_SUCCESS))
9718	{
9719	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9720
9721	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
9722	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9723	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9724	{
9725	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9726	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9727	EMInterpretWrmsr() changes it. */
9728	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9729	}
9730	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9731	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9732	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9733	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9734
9735	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9736	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9737	{
9738	switch (pMixedCtx->ecx)
9739	{
9740	case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9741	case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9742	case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9743	case MSR_K8_FS_BASE: /* no break */
9744	case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9745	case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9746	}
9747	}
9748	#ifdef VBOX_STRICT
9749	else
9750	{
9751	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9752	switch (pMixedCtx->ecx)
9753	{
9754	case MSR_IA32_SYSENTER_CS:
9755	case MSR_IA32_SYSENTER_EIP:
9756	case MSR_IA32_SYSENTER_ESP:
9757	case MSR_K8_FS_BASE:
9758	case MSR_K8_GS_BASE:
9759	{
9760	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9761	HMVMX_RETURN_UNEXPECTED_EXIT();
9762	}
9763
9764	case MSR_K8_LSTAR:
9765	case MSR_K6_STAR:
9766	case MSR_K8_SF_MASK:
9767	case MSR_K8_TSC_AUX:
9768	case MSR_K8_KERNEL_GS_BASE:
9769	{
9770	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9771	pMixedCtx->ecx));
9772	HMVMX_RETURN_UNEXPECTED_EXIT();
9773	}
9774	}
9775	}
9776	#endif /* VBOX_STRICT */
9777	}
9778	return rc;
9779	}
9780
9781
9782	/**
9783	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9784	*/
9785	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9786	{
9787	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9788
9789	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9791	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9792	return VERR_EM_INTERPRETER;
9793	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9794	HMVMX_RETURN_UNEXPECTED_EXIT();
9795	}
9796
9797
9798	/**
9799	* VM-exit handler for when the TPR value is lowered below the specified
9800	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9801	*/
9802	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9803	{
9804	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9805	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9806
9807	/*
9808	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9809	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9810	* resume guest execution.
9811	*/
9812	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9813	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9814	return VINF_SUCCESS;
9815	}
9816
9817
9818	/**
9819	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9820	* VM-exit.
9821	*
9822	* @retval VINF_SUCCESS when guest execution can continue.
9823	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9824	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9825	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9826	* recompiler.
9827	*/
9828	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9829	{
9830	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9831	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9832	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9833	AssertRCReturn(rc, rc);
9834
9835	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9836	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9837	PVM pVM = pVCpu->CTX_SUFF(pVM);
9838	switch (uAccessType)
9839	{
9840	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9841	{
9842	#if 0
9843	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9844	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9845	#else
9846	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9847	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9848	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9849	#endif
9850	AssertRCReturn(rc, rc);
9851
9852	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9853	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9854	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9855	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
9856
9857	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9858	{
9859	case 0: /* CR0 */
9860	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9861	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9862	break;
9863	case 2: /* CR2 */
9864	/* Nothing to do here, CR2 it's not part of the VMCS. */
9865	break;
9866	case 3: /* CR3 */
9867	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9868	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9869	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9870	break;
9871	case 4: /* CR4 */
9872	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9873	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9874	break;
9875	case 8: /* CR8 */
9876	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9877	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9878	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9879	break;
9880	default:
9881	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9882	break;
9883	}
9884
9885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9886	break;
9887	}
9888
9889	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9890	{
9891	/* EMInterpretCRxRead() requires EFER MSR, CS. */
9892	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9893	AssertRCReturn(rc, rc);
9894	Assert( !pVM->hm.s.fNestedPaging
9895	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9896	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9897
9898	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9899	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9900	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9901
9902	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9903	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9904	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9905	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
9906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9907	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9908	break;
9909	}
9910
9911	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9912	{
9913	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9914	AssertRCReturn(rc, rc);
9915	rc = EMInterpretCLTS(pVM, pVCpu);
9916	AssertRCReturn(rc, rc);
9917	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9918	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9919	Log4(("CRX CLTS write rc=%d\n", rc));
9920	break;
9921	}
9922
9923	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9924	{
9925	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9926	AssertRCReturn(rc, rc);
9927	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9928	if (RT_LIKELY(rc == VINF_SUCCESS))
9929	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9930	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9931	Log4(("CRX LMSW write rc=%d\n", rc));
9932	break;
9933	}
9934
9935	default:
9936	{
9937	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9938	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9939	}
9940	}
9941
9942	/* Validate possible error codes. */
9943	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
9944	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9945	if (RT_SUCCESS(rc))
9946	{
9947	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9948	AssertRCReturn(rc2, rc2);
9949	}
9950
9951	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9952	return rc;
9953	}
9954
9955
9956	/**
9957	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9958	* VM-exit.
9959	*/
9960	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9961	{
9962	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9963	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9964
9965	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9966	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9967	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9968	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9969	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9970	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9971	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9972	AssertRCReturn(rc2, rc2);
9973
9974	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9975	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9976	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9977	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9978	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9979	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9980	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9981
9982	/* I/O operation lookup arrays. */
9983	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9984	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9985
9986	VBOXSTRICTRC rcStrict;
9987	const uint32_t cbValue = s_aIOSizes[uIOWidth];
9988	const uint32_t cbInstr = pVmxTransient->cbInstr;
9989	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9990	PVM pVM = pVCpu->CTX_SUFF(pVM);
9991	if (fIOString)
9992	{
9993	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158}*/
9994	/*
9995	* INS/OUTS - I/O String instruction.
9996	*
9997	* Use instruction-information if available, otherwise fall back on
9998	* interpreting the instruction.
9999	*/
10000	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10001	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10002	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10003	{
10004	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVCpu, pVmxTransient);
10005	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10006	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10007	AssertRCReturn(rc2, rc2);
10008	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10009	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10010	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10011	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10012	if (fIOWrite)
10013	{
10014	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10015	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10016	}
10017	else
10018	{
10019	/*
10020	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10021	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10022	* See Intel Instruction spec. for "INS".
10023	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10024	*/
10025	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10026	}
10027	}
10028	else
10029	{
10030	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10031	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10032	AssertRCReturn(rc2, rc2);
10033	rcStrict = IEMExecOne(pVCpu);
10034	}
10035	/** @todo IEM needs to be setting these flags somehow. */
10036	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10037	fUpdateRipAlready = true;
10038	#else
10039	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10040	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
10041	if (RT_SUCCESS(rcStrict))
10042	{
10043	if (fIOWrite)
10044	{
10045	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10046	(DISCPUMODE)pDis->uAddrMode, cbValue);
10047	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10048	}
10049	else
10050	{
10051	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10052	(DISCPUMODE)pDis->uAddrMode, cbValue);
10053	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10054	}
10055	}
10056	else
10057	{
10058	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10059	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10060	}
10061	#endif
10062	}
10063	else
10064	{
10065	/*
10066	* IN/OUT - I/O instruction.
10067	*/
10068	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10069	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10070	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10071	if (fIOWrite)
10072	{
10073	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10074	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10075	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10076	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10077	}
10078	else
10079	{
10080	uint32_t u32Result = 0;
10081	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10082	if (IOM_SUCCESS(rcStrict))
10083	{
10084	/* Save result of I/O IN instr. in AL/AX/EAX. */
10085	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10086	}
10087	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10088	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10090	}
10091	}
10092
10093	if (IOM_SUCCESS(rcStrict))
10094	{
10095	if (!fUpdateRipAlready)
10096	{
10097	pMixedCtx->rip += cbInstr;
10098	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10099	}
10100
10101	/* INS & OUTS with REP prefix modify RFLAGS. */
10102	if (fIOString)
10103	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10104
10105	/*
10106	* If any I/O breakpoints are armed, we need to check if one triggered
10107	* and take appropriate action.
10108	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10109	*/
10110	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10111	AssertRCReturn(rc2, rc2);
10112
10113	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10114	* execution engines about whether hyper BPs and such are pending. */
10115	uint32_t const uDr7 = pMixedCtx->dr[7];
10116	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10117	&& X86_DR7_ANY_RW_IO(uDr7)
10118	&& (pMixedCtx->cr4 & X86_CR4_DE))
10119	\|\| DBGFBpIsHwIoArmed(pVM)))
10120	{
10121	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10122
10123	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10124	VMMRZCallRing3Disable(pVCpu);
10125	HM_DISABLE_PREEMPT_IF_NEEDED();
10126
10127	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10128
10129	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10130	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10131	{
10132	/* Raise #DB. */
10133	if (fIsGuestDbgActive)
10134	ASMSetDR6(pMixedCtx->dr[6]);
10135	if (pMixedCtx->dr[7] != uDr7)
10136	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10137
10138	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10139	}
10140	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10141	else if ( rcStrict2 != VINF_SUCCESS
10142	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10143	rcStrict = rcStrict2;
10144
10145	HM_RESTORE_PREEMPT_IF_NEEDED();
10146	VMMRZCallRing3Enable(pVCpu);
10147	}
10148	}
10149
10150	#ifdef DEBUG
10151	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10152	Assert(!fIOWrite);
10153	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10154	Assert(fIOWrite);
10155	else
10156	{
10157	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10158	* statuses, that the VMM device and some others may return. See
10159	* IOM_SUCCESS() for guidance. */
10160	AssertMsg( RT_FAILURE(rcStrict)
10161	\|\| rcStrict == VINF_SUCCESS
10162	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10163	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10164	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10165	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10166	}
10167	#endif
10168
10169	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10170	return VBOXSTRICTRC_TODO(rcStrict);
10171	}
10172
10173
10174	/**
10175	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10176	* VM-exit.
10177	*/
10178	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10179	{
10180	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10181
10182	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10183	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10184	AssertRCReturn(rc, rc);
10185	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10186	{
10187	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10188	AssertRCReturn(rc, rc);
10189	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10190	{
10191	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10192
10193	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10194	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10195	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10196	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10197	{
10198	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10199	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10200
10201	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10202	Assert(!pVCpu->hm.s.Event.fPending);
10203	pVCpu->hm.s.Event.fPending = true;
10204	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10205	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10206	AssertRCReturn(rc, rc);
10207	if (fErrorCodeValid)
10208	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10209	else
10210	pVCpu->hm.s.Event.u32ErrCode = 0;
10211	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10212	&& uVector == X86_XCPT_PF)
10213	{
10214	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10215	}
10216
10217	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10218	}
10219	}
10220	}
10221
10222	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10223	* emulation. */
10224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10225	return VERR_EM_INTERPRETER;
10226	}
10227
10228
10229	/**
10230	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10231	*/
10232	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10233	{
10234	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10235	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10236	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10237	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10238	AssertRCReturn(rc, rc);
10239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10240	return VINF_EM_DBG_STEPPED;
10241	}
10242
10243
10244	/**
10245	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10246	*/
10247	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10248	{
10249	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10250
10251	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10252	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10253	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10254	return VINF_SUCCESS;
10255	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10256	return rc;
10257
10258	#if 0
10259	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10260	* just sync the whole thing. */
10261	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10262	#else
10263	/* Aggressive state sync. for now. */
10264	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10265	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10266	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10267	#endif
10268	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10269	AssertRCReturn(rc, rc);
10270
10271	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10272	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10273	switch (uAccessType)
10274	{
10275	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10276	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10277	{
10278	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10279	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10280	{
10281	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10282	}
10283
10284	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10285	GCPhys &= PAGE_BASE_GC_MASK;
10286	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10287	PVM pVM = pVCpu->CTX_SUFF(pVM);
10288	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10289	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10290
10291	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10292	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10293	CPUMCTX2CORE(pMixedCtx), GCPhys);
10294	rc = VBOXSTRICTRC_VAL(rc2);
10295	Log4(("ApicAccess rc=%d\n", rc));
10296	if ( rc == VINF_SUCCESS
10297	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10298	\|\| rc == VERR_PAGE_NOT_PRESENT)
10299	{
10300	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10301	\| HM_CHANGED_GUEST_RSP
10302	\| HM_CHANGED_GUEST_RFLAGS
10303	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10304	rc = VINF_SUCCESS;
10305	}
10306	break;
10307	}
10308
10309	default:
10310	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10311	rc = VINF_EM_RAW_EMULATE_INSTR;
10312	break;
10313	}
10314
10315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10316	return rc;
10317	}
10318
10319
10320	/**
10321	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10322	* VM-exit.
10323	*/
10324	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10325	{
10326	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10327
10328	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10329	if (pVmxTransient->fWasGuestDebugStateActive)
10330	{
10331	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10332	HMVMX_RETURN_UNEXPECTED_EXIT();
10333	}
10334
10335	int rc = VERR_INTERNAL_ERROR_5;
10336	if ( !DBGFIsStepping(pVCpu)
10337	&& !pVCpu->hm.s.fSingleInstruction
10338	&& !pVmxTransient->fWasHyperDebugStateActive)
10339	{
10340	/* Don't intercept MOV DRx and #DB any more. */
10341	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10342	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10343	AssertRCReturn(rc, rc);
10344
10345	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10346	{
10347	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10348	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10349	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10350	AssertRCReturn(rc, rc);
10351	#endif
10352	}
10353
10354	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10355	VMMRZCallRing3Disable(pVCpu);
10356	HM_DISABLE_PREEMPT_IF_NEEDED();
10357
10358	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10359	PVM pVM = pVCpu->CTX_SUFF(pVM);
10360	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10361	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10362
10363	HM_RESTORE_PREEMPT_IF_NEEDED();
10364	VMMRZCallRing3Enable(pVCpu);
10365
10366	#ifdef VBOX_WITH_STATISTICS
10367	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10368	AssertRCReturn(rc, rc);
10369	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10370	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10371	else
10372	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10373	#endif
10374	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10375	return VINF_SUCCESS;
10376	}
10377
10378	/*
10379	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10380	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10381	*/
10382	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10383	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10384	AssertRCReturn(rc, rc);
10385	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10386
10387	PVM pVM = pVCpu->CTX_SUFF(pVM);
10388	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10389	{
10390	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10391	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10392	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10393	if (RT_SUCCESS(rc))
10394	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10396	}
10397	else
10398	{
10399	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10400	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10401	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10402	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10403	}
10404
10405	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10406	if (RT_SUCCESS(rc))
10407	{
10408	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10409	AssertRCReturn(rc2, rc2);
10410	}
10411	return rc;
10412	}
10413
10414
10415	/**
10416	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10417	* Conditional VM-exit.
10418	*/
10419	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10420	{
10421	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10422	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10423
10424	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10425	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10426	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10427	return VINF_SUCCESS;
10428	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10429	return rc;
10430
10431	RTGCPHYS GCPhys = 0;
10432	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10433
10434	#if 0
10435	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10436	#else
10437	/* Aggressive state sync. for now. */
10438	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10439	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10440	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10441	#endif
10442	AssertRCReturn(rc, rc);
10443
10444	/*
10445	* If we succeed, resume guest execution.
10446	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10447	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10448	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10449	* weird case. See @bugref{6043}.
10450	*/
10451	PVM pVM = pVCpu->CTX_SUFF(pVM);
10452	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10453	rc = VBOXSTRICTRC_VAL(rc2);
10454	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10455	if ( rc == VINF_SUCCESS
10456	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10457	\|\| rc == VERR_PAGE_NOT_PRESENT)
10458	{
10459	/* Successfully handled MMIO operation. */
10460	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10461	\| HM_CHANGED_GUEST_RSP
10462	\| HM_CHANGED_GUEST_RFLAGS
10463	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10464	rc = VINF_SUCCESS;
10465	}
10466	return rc;
10467	}
10468
10469
10470	/**
10471	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10472	* VM-exit.
10473	*/
10474	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10475	{
10476	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10477	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10478
10479	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10480	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10481	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10482	return VINF_SUCCESS;
10483	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10484	return rc;
10485
10486	RTGCPHYS GCPhys = 0;
10487	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10488	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10489	#if 0
10490	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10491	#else
10492	/* Aggressive state sync. for now. */
10493	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10494	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10495	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10496	#endif
10497	AssertRCReturn(rc, rc);
10498
10499	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10500	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10501
10502	RTGCUINT uErrorCode = 0;
10503	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10504	uErrorCode \|= X86_TRAP_PF_ID;
10505	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10506	uErrorCode \|= X86_TRAP_PF_RW;
10507	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10508	uErrorCode \|= X86_TRAP_PF_P;
10509
10510	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10511
10512	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10513	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10514
10515	/* Handle the pagefault trap for the nested shadow table. */
10516	PVM pVM = pVCpu->CTX_SUFF(pVM);
10517	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10518	TRPMResetTrap(pVCpu);
10519
10520	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10521	if ( rc == VINF_SUCCESS
10522	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10523	\|\| rc == VERR_PAGE_NOT_PRESENT)
10524	{
10525	/* Successfully synced our nested page tables. */
10526	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10527	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10528	\| HM_CHANGED_GUEST_RSP
10529	\| HM_CHANGED_GUEST_RFLAGS);
10530	return VINF_SUCCESS;
10531	}
10532
10533	Log4(("EPT return to ring-3 rc=%d\n"));
10534	return rc;
10535	}
10536
10537	/** @} */
10538
10539	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10540	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10541	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10542
10543	/** @name VM-exit exception handlers.
10544	* @{
10545	*/
10546
10547	/**
10548	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10549	*/
10550	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10551	{
10552	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10553	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10554
10555	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10556	AssertRCReturn(rc, rc);
10557
10558	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10559	{
10560	/* Old-style FPU error reporting needs some extra work. */
10561	/** @todo don't fall back to the recompiler, but do it manually. */
10562	return VERR_EM_INTERPRETER;
10563	}
10564
10565	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10566	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10567	return rc;
10568	}
10569
10570
10571	/**
10572	* VM-exit exception handler for #BP (Breakpoint exception).
10573	*/
10574	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10575	{
10576	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10578
10579	/** @todo Try optimize this by not saving the entire guest state unless
10580	* really needed. */
10581	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10582	AssertRCReturn(rc, rc);
10583
10584	PVM pVM = pVCpu->CTX_SUFF(pVM);
10585	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10586	if (rc == VINF_EM_RAW_GUEST_TRAP)
10587	{
10588	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10589	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10590	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10591	AssertRCReturn(rc, rc);
10592
10593	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10594	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10595	}
10596
10597	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10598	return rc;
10599	}
10600
10601
10602	/**
10603	* VM-exit exception handler for #DB (Debug exception).
10604	*/
10605	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10606	{
10607	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10609	Log6(("XcptDB\n"));
10610
10611	/*
10612	* Get the DR6-like values from the exit qualification and pass it to DBGF
10613	* for processing.
10614	*/
10615	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10616	AssertRCReturn(rc, rc);
10617
10618	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10619	uint64_t uDR6 = X86_DR6_INIT_VAL;
10620	uDR6 \|= ( pVmxTransient->uExitQualification
10621	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10622
10623	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10624	if (rc == VINF_EM_RAW_GUEST_TRAP)
10625	{
10626	/*
10627	* The exception was for the guest. Update DR6, DR7.GD and
10628	* IA32_DEBUGCTL.LBR before forwarding it.
10629	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10630	*/
10631	VMMRZCallRing3Disable(pVCpu);
10632	HM_DISABLE_PREEMPT_IF_NEEDED();
10633
10634	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10635	pMixedCtx->dr[6] \|= uDR6;
10636	if (CPUMIsGuestDebugStateActive(pVCpu))
10637	ASMSetDR6(pMixedCtx->dr[6]);
10638
10639	HM_RESTORE_PREEMPT_IF_NEEDED();
10640	VMMRZCallRing3Enable(pVCpu);
10641
10642	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10643	AssertRCReturn(rc, rc);
10644
10645	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10646	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10647
10648	/* Paranoia. */
10649	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10650	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10651
10652	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10653	AssertRCReturn(rc, rc);
10654
10655	/*
10656	* Raise #DB in the guest.
10657	*/
10658	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10659	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10660	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10661	AssertRCReturn(rc, rc);
10662	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10663	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10664	return VINF_SUCCESS;
10665	}
10666
10667	/*
10668	* Not a guest trap, must be a hypervisor related debug event then.
10669	* Update DR6 in case someone is interested in it.
10670	*/
10671	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10672	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10673	CPUMSetHyperDR6(pVCpu, uDR6);
10674
10675	return rc;
10676	}
10677
10678
10679	/**
10680	* VM-exit exception handler for #NM (Device-not-available exception: floating
10681	* point exception).
10682	*/
10683	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10684	{
10685	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10686
10687	/* We require CR0 and EFER. EFER is always up-to-date. */
10688	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10689	AssertRCReturn(rc, rc);
10690
10691	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10692	VMMRZCallRing3Disable(pVCpu);
10693	HM_DISABLE_PREEMPT_IF_NEEDED();
10694
10695	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10696	if (pVmxTransient->fWasGuestFPUStateActive)
10697	{
10698	rc = VINF_EM_RAW_GUEST_TRAP;
10699	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10700	}
10701	else
10702	{
10703	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10704	Assert(!pVmxTransient->fWasGuestFPUStateActive);
10705	#endif
10706	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10707	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10708	}
10709
10710	HM_RESTORE_PREEMPT_IF_NEEDED();
10711	VMMRZCallRing3Enable(pVCpu);
10712
10713	if (rc == VINF_SUCCESS)
10714	{
10715	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
10716	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10717	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10718	pVCpu->hm.s.fUseGuestFpu = true;
10719	}
10720	else
10721	{
10722	/* Forward #NM to the guest. */
10723	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10724	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10725	AssertRCReturn(rc, rc);
10726	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10727	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
10728	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10729	}
10730
10731	return VINF_SUCCESS;
10732	}
10733
10734
10735	/**
10736	* VM-exit exception handler for #GP (General-protection exception).
10737	*
10738	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
10739	*/
10740	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10741	{
10742	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10743	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10744
10745	int rc = VERR_INTERNAL_ERROR_5;
10746	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10747	{
10748	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10749	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10750	rc = hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
10751	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10752	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10753	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10754	AssertRCReturn(rc, rc);
10755	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
10756	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10757	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10758	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10759	return rc;
10760	#else
10761	/* We don't intercept #GP. */
10762	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10763	return VERR_VMX_UNEXPECTED_EXCEPTION;
10764	#endif
10765	}
10766
10767	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10768	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10769
10770	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10771	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10772	AssertRCReturn(rc, rc);
10773
10774	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10775	uint32_t cbOp = 0;
10776	PVM pVM = pVCpu->CTX_SUFF(pVM);
10777	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10778	if (RT_SUCCESS(rc))
10779	{
10780	rc = VINF_SUCCESS;
10781	Assert(cbOp == pDis->cbInstr);
10782	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10783	switch (pDis->pCurInstr->uOpcode)
10784	{
10785	case OP_CLI:
10786	{
10787	pMixedCtx->eflags.Bits.u1IF = 0;
10788	pMixedCtx->rip += pDis->cbInstr;
10789	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10791	break;
10792	}
10793
10794	case OP_STI:
10795	{
10796	pMixedCtx->eflags.Bits.u1IF = 1;
10797	pMixedCtx->rip += pDis->cbInstr;
10798	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10799	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10800	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
10801	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10802	break;
10803	}
10804
10805	case OP_HLT:
10806	{
10807	rc = VINF_EM_HALT;
10808	pMixedCtx->rip += pDis->cbInstr;
10809	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10811	break;
10812	}
10813
10814	case OP_POPF:
10815	{
10816	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10817	uint32_t cbParm = 0;
10818	uint32_t uMask = 0;
10819	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10820	{
10821	cbParm = 4;
10822	uMask = 0xffffffff;
10823	}
10824	else
10825	{
10826	cbParm = 2;
10827	uMask = 0xffff;
10828	}
10829
10830	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
10831	RTGCPTR GCPtrStack = 0;
10832	X86EFLAGS Eflags;
10833	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10834	&GCPtrStack);
10835	if (RT_SUCCESS(rc))
10836	{
10837	Assert(sizeof(Eflags.u32) >= cbParm);
10838	Eflags.u32 = 0;
10839	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10840	}
10841	if (RT_FAILURE(rc))
10842	{
10843	rc = VERR_EM_INTERPRETER;
10844	break;
10845	}
10846	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10847	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10848	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10849	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10850	pMixedCtx->esp += cbParm;
10851	pMixedCtx->esp &= uMask;
10852	pMixedCtx->rip += pDis->cbInstr;
10853
10854	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10855	\| HM_CHANGED_GUEST_RSP
10856	\| HM_CHANGED_GUEST_RFLAGS);
10857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10858	break;
10859	}
10860
10861	case OP_PUSHF:
10862	{
10863	uint32_t cbParm = 0;
10864	uint32_t uMask = 0;
10865	if (pDis->fPrefix & DISPREFIX_OPSIZE)
10866	{
10867	cbParm = 4;
10868	uMask = 0xffffffff;
10869	}
10870	else
10871	{
10872	cbParm = 2;
10873	uMask = 0xffff;
10874	}
10875
10876	/* Get the stack pointer & push the contents of eflags onto the stack. */
10877	RTGCPTR GCPtrStack = 0;
10878	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10879	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10880	if (RT_FAILURE(rc))
10881	{
10882	rc = VERR_EM_INTERPRETER;
10883	break;
10884	}
10885	X86EFLAGS Eflags = pMixedCtx->eflags;
10886	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10887	Eflags.Bits.u1RF = 0;
10888	Eflags.Bits.u1VM = 0;
10889
10890	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10891	if (RT_FAILURE(rc))
10892	{
10893	rc = VERR_EM_INTERPRETER;
10894	break;
10895	}
10896	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10897	pMixedCtx->esp -= cbParm;
10898	pMixedCtx->esp &= uMask;
10899	pMixedCtx->rip += pDis->cbInstr;
10900	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
10901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10902	break;
10903	}
10904
10905	case OP_IRET:
10906	{
10907	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10908	* instruction reference. */
10909	RTGCPTR GCPtrStack = 0;
10910	uint32_t uMask = 0xffff;
10911	uint16_t aIretFrame[3];
10912	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
10913	{
10914	rc = VERR_EM_INTERPRETER;
10915	break;
10916	}
10917	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10918	&GCPtrStack);
10919	if (RT_SUCCESS(rc))
10920	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10921	if (RT_FAILURE(rc))
10922	{
10923	rc = VERR_EM_INTERPRETER;
10924	break;
10925	}
10926	pMixedCtx->eip = 0;
10927	pMixedCtx->ip = aIretFrame[0];
10928	pMixedCtx->cs.Sel = aIretFrame[1];
10929	pMixedCtx->cs.ValidSel = aIretFrame[1];
10930	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10931	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10932	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10933	pMixedCtx->sp += sizeof(aIretFrame);
10934	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10935	\| HM_CHANGED_GUEST_SEGMENT_REGS
10936	\| HM_CHANGED_GUEST_RSP
10937	\| HM_CHANGED_GUEST_RFLAGS);
10938	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10940	break;
10941	}
10942
10943	case OP_INT:
10944	{
10945	uint16_t uVector = pDis->Param1.uValue & 0xff;
10946	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10948	break;
10949	}
10950
10951	case OP_INTO:
10952	{
10953	if (pMixedCtx->eflags.Bits.u1OF)
10954	{
10955	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10957	}
10958	break;
10959	}
10960
10961	default:
10962	{
10963	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10964	EMCODETYPE_SUPERVISOR);
10965	rc = VBOXSTRICTRC_VAL(rc2);
10966	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10967	Log4(("#GP rc=%Rrc\n", rc));
10968	break;
10969	}
10970	}
10971	}
10972	else
10973	rc = VERR_EM_INTERPRETER;
10974
10975	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
10976	("#GP Unexpected rc=%Rrc\n", rc));
10977	return rc;
10978	}
10979
10980
10981	/**
10982	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10983	* the exception reported in the VMX transient structure back into the VM.
10984	*
10985	* @remarks Requires uExitIntInfo in the VMX transient structure to be
10986	* up-to-date.
10987	*/
10988	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10989	{
10990	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10991
10992	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10993	hmR0VmxCheckExitDueToEventDelivery(). */
10994	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
10995	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10996	AssertRCReturn(rc, rc);
10997	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10998
10999	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11000	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11001	return VINF_SUCCESS;
11002	}
11003
11004
11005	/**
11006	* VM-exit exception handler for #PF (Page-fault exception).
11007	*/
11008	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11009	{
11010	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11011	PVM pVM = pVCpu->CTX_SUFF(pVM);
11012	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11013	rc \|= hmR0VmxReadExitIntInfoVmcs(pVCpu, pVmxTransient);
11014	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVCpu, pVmxTransient);
11015	AssertRCReturn(rc, rc);
11016
11017	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11018	if (pVM->hm.s.fNestedPaging)
11019	{
11020	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11021	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11022	{
11023	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11024	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11025	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11026	}
11027	else
11028	{
11029	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11030	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11031	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11032	}
11033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11034	return rc;
11035	}
11036	#else
11037	Assert(!pVM->hm.s.fNestedPaging);
11038	#endif
11039
11040	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11041	AssertRCReturn(rc, rc);
11042
11043	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11044	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11045
11046	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11047	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11048	(RTGCPTR)pVmxTransient->uExitQualification);
11049
11050	Log4(("#PF: rc=%Rrc\n", rc));
11051	if (rc == VINF_SUCCESS)
11052	{
11053	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11054	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11055	* memory? We don't update the whole state here... */
11056	VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11057	\| HM_CHANGED_GUEST_RSP
11058	\| HM_CHANGED_GUEST_RFLAGS
11059	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11060	TRPMResetTrap(pVCpu);
11061	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11062	return rc;
11063	}
11064	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11065	{
11066	if (!pVmxTransient->fVectoringPF)
11067	{
11068	/* It's a guest page fault and needs to be reflected to the guest. */
11069	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11070	TRPMResetTrap(pVCpu);
11071	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11072	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11073	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11074	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11075	}
11076	else
11077	{
11078	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11079	TRPMResetTrap(pVCpu);
11080	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11081	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11082	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11083	}
11084
11085	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11086	return VINF_SUCCESS;
11087	}
11088
11089	TRPMResetTrap(pVCpu);
11090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11091	return rc;
11092	}
11093
11094	/** @} */
11095

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49284

Download in other formats: