HMVMXR0.cpp@ 50271

Last change on this file since 50271 was 50271, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Comment clarification.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 472.1 KB

Line
1	/* $Id: HMVMXR0.cpp 50271 2014-01-29 14:25:25Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if (pVM->hm.s.fAllow64BitGuests)
2336	{
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	}
2342	#endif
2343	}
2344
2345	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2346	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2347	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2348
2349	if ((val & zap) != val)
2350	{
2351	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2352	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set processor-based VM-execution controls. */
2361	pVCpu->hm.s.vmx.u32ProcCtls = val;
2362
2363	/*
2364	* Secondary processor-based VM-execution controls.
2365	*/
2366	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2367	{
2368	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2373
2374	if (pVM->hm.s.fNestedPaging)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2376	else
2377	{
2378	/*
2379	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2380	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2381	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2382	*/
2383	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2385	}
2386
2387	if (pVM->hm.s.vmx.fVpid)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2389
2390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2392
2393	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2394	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2395	* done dynamically. */
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2397	{
2398	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2399	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2401	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2402	AssertRCReturn(rc, rc);
2403	}
2404
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2411	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2412	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2413	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2414	}
2415
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2417	AssertRCReturn(rc, rc);
2418
2419	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2420	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2421	}
2422	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2423	{
2424	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2425	"available\n"));
2426	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	return VINF_SUCCESS;
2431	}
2432
2433
2434	/**
2435	* Sets up miscellaneous (everything other than Pin & Processor-based
2436	* VM-execution) control fields in the VMCS.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	*/
2442	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2443	{
2444	NOREF(pVM);
2445	AssertPtr(pVM);
2446	AssertPtr(pVCpu);
2447
2448	int rc = VERR_GENERAL_FAILURE;
2449
2450	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2451	#if 0
2452	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2455
2456	/*
2457	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2458	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2459	* We thus use the exception bitmap to control it rather than use both.
2460	*/
2461	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2463
2464	/** @todo Explore possibility of using IO-bitmaps. */
2465	/* All IO & IOIO instructions cause VM-exits. */
2466	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/* Initialize the MSR-bitmap area. */
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2473	#endif
2474
2475	/* Setup MSR auto-load/store area. */
2476	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2477	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2479	AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2481	AssertRCReturn(rc, rc);
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2486	AssertRCReturn(rc, rc);
2487
2488	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2489	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2490	AssertRCReturn(rc, rc);
2491
2492	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2493	#if 0
2494	/* Setup debug controls */
2495	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2498	AssertRCReturn(rc, rc);
2499	#endif
2500
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Sets up the initial exception bitmap in the VMCS based on static conditions
2507	* (i.e. conditions that cannot ever change after starting the VM).
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	*/
2513	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2514	{
2515	AssertPtr(pVM);
2516	AssertPtr(pVCpu);
2517
2518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2519
2520	uint32_t u32XcptBitmap = 0;
2521
2522	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2523	if (!pVM->hm.s.fNestedPaging)
2524	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2525
2526	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2527	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2528	AssertRCReturn(rc, rc);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Sets up the initial guest-state mask. The guest-state mask is consulted
2535	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2536	* for the nested virtualization case (as it would cause a VM-exit).
2537	*
2538	* @param pVCpu Pointer to the VMCPU.
2539	*/
2540	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2541	{
2542	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2543	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2544	return VINF_SUCCESS;
2545	}
2546
2547
2548	/**
2549	* Does per-VM VT-x initialization.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	*/
2554	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2555	{
2556	LogFlowFunc(("pVM=%p\n", pVM));
2557
2558	int rc = hmR0VmxStructsAlloc(pVM);
2559	if (RT_FAILURE(rc))
2560	{
2561	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2562	return rc;
2563	}
2564
2565	return VINF_SUCCESS;
2566	}
2567
2568
2569	/**
2570	* Does per-VM VT-x termination.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM Pointer to the VM.
2574	*/
2575	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2576	{
2577	LogFlowFunc(("pVM=%p\n", pVM));
2578
2579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2580	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2581	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2582	#endif
2583	hmR0VmxStructsFree(pVM);
2584	return VINF_SUCCESS;
2585	}
2586
2587
2588	/**
2589	* Sets up the VM for execution under VT-x.
2590	* This function is only called once per-VM during initialization.
2591	*
2592	* @returns VBox status code.
2593	* @param pVM Pointer to the VM.
2594	*/
2595	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2596	{
2597	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2599
2600	LogFlowFunc(("pVM=%p\n", pVM));
2601
2602	/*
2603	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2604	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2605	*/
2606	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2607	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2608	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2609	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2610	{
2611	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2612	return VERR_INTERNAL_ERROR;
2613	}
2614
2615	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2616	/*
2617	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2618	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2619	*/
2620	if ( pVM->hm.s.fAllow64BitGuests
2621	&& !HMVMX_IS_64BIT_HOST_MODE())
2622	{
2623	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2624	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2625	}
2626	#endif
2627
2628	/* Initialize these always, see hmR3InitFinalizeR0().*/
2629	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2630	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2631
2632	/* Setup the tagged-TLB flush handlers. */
2633	int rc = hmR0VmxSetupTaggedTlb(pVM);
2634	if (RT_FAILURE(rc))
2635	{
2636	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2637	return rc;
2638	}
2639
2640	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2641	{
2642	PVMCPU pVCpu = &pVM->aCpus[i];
2643	AssertPtr(pVCpu);
2644	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2645
2646	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2647	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2648
2649	/* Set revision dword at the beginning of the VMCS structure. */
2650	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2651
2652	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2653	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2654	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2655	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2656
2657	/* Load this VMCS as the current VMCS. */
2658	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2659	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2660	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2661
2662	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2671	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2672	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2673
2674	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2675	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2676	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2677
2678	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2679	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2680	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2681
2682	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2683	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686	#endif
2687
2688	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2689	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2690	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2694
2695	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2696	}
2697
2698	return VINF_SUCCESS;
2699	}
2700
2701
2702	/**
2703	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2704	* the VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM Pointer to the VM.
2708	* @param pVCpu Pointer to the VMCPU.
2709	*/
2710	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2711	{
2712	NOREF(pVM); NOREF(pVCpu);
2713
2714	RTCCUINTREG uReg = ASMGetCR0();
2715	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2716	AssertRCReturn(rc, rc);
2717
2718	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2720	if (HMVMX_IS_64BIT_HOST_MODE())
2721	{
2722	uint64_t uRegCR3 = HMR0Get64bitCR3();
2723	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2724	}
2725	else
2726	#endif
2727	{
2728	uReg = ASMGetCR3();
2729	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2730	}
2731	AssertRCReturn(rc, rc);
2732
2733	uReg = ASMGetCR4();
2734	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2735	AssertRCReturn(rc, rc);
2736	return rc;
2737	}
2738
2739
2740	#if HC_ARCH_BITS == 64
2741	/**
2742	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2743	* requirements. See hmR0VmxSaveHostSegmentRegs().
2744	*/
2745	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2746	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2747	{ \
2748	bool fValidSelector = true; \
2749	if ((selValue) & X86_SEL_LDT) \
2750	{ \
2751	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2752	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2753	} \
2754	if (fValidSelector) \
2755	{ \
2756	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2757	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2758	} \
2759	(selValue) = 0; \
2760	}
2761	#endif
2762
2763
2764	/**
2765	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2766	* the host-state area in the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVM Pointer to the VM.
2770	* @param pVCpu Pointer to the VMCPU.
2771	*/
2772	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2773	{
2774	NOREF(pVM);
2775	int rc = VERR_INTERNAL_ERROR_5;
2776
2777	/*
2778	* Host DS, ES, FS and GS segment registers.
2779	*/
2780	#if HC_ARCH_BITS == 64
2781	RTSEL uSelDS = ASMGetDS();
2782	RTSEL uSelES = ASMGetES();
2783	RTSEL uSelFS = ASMGetFS();
2784	RTSEL uSelGS = ASMGetGS();
2785	#else
2786	RTSEL uSelDS = 0;
2787	RTSEL uSelES = 0;
2788	RTSEL uSelFS = 0;
2789	RTSEL uSelGS = 0;
2790	#endif
2791
2792	/* Recalculate which host-state bits need to be manually restored. */
2793	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2794
2795	/*
2796	* Host CS and SS segment registers.
2797	*/
2798	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2799	RTSEL uSelCS;
2800	RTSEL uSelSS;
2801	if (HMVMX_IS_64BIT_HOST_MODE())
2802	{
2803	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2804	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2805	}
2806	else
2807	{
2808	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2809	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2810	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2811	}
2812	#else
2813	RTSEL uSelCS = ASMGetCS();
2814	RTSEL uSelSS = ASMGetSS();
2815	#endif
2816
2817	/*
2818	* Host TR segment register.
2819	*/
2820	RTSEL uSelTR = ASMGetTR();
2821
2822	#if HC_ARCH_BITS == 64
2823	/*
2824	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2825	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2826	*/
2827	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2828	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2829	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2830	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2831	# undef VMXLOCAL_ADJUST_HOST_SEG
2832	#endif
2833
2834	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2835	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2836	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2837	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2838	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2839	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2840	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2841	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2842	Assert(uSelCS);
2843	Assert(uSelTR);
2844
2845	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2846	#if 0
2847	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2848	Assert(uSelSS != 0);
2849	#endif
2850
2851	/* Write these host selector fields into the host-state area in the VMCS. */
2852	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2853	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2854	#if HC_ARCH_BITS == 64
2855	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2857	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2859	#endif
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2861
2862	/*
2863	* Host GDTR and IDTR.
2864	*/
2865	RTGDTR Gdtr;
2866	RT_ZERO(Gdtr);
2867	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2868	if (HMVMX_IS_64BIT_HOST_MODE())
2869	{
2870	X86XDTR64 Gdtr64;
2871	X86XDTR64 Idtr64;
2872	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2873	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2874	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2875
2876	Gdtr.cbGdt = Gdtr64.cb;
2877	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2878	}
2879	else
2880	#endif
2881	{
2882	RTIDTR Idtr;
2883	ASMGetGDTR(&Gdtr);
2884	ASMGetIDTR(&Idtr);
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2887
2888	#if HC_ARCH_BITS == 64
2889	/*
2890	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2891	* maximum limit (0xffff) on every VM-exit.
2892	*/
2893	if (Gdtr.cbGdt != 0xffff)
2894	{
2895	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2896	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2897	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2898	}
2899
2900	/*
2901	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2902	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2903	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2904	*/
2905	if (Idtr.cbIdt < 0x0fff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2908	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2910	}
2911	#endif
2912	}
2913
2914	/*
2915	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2916	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2917	*/
2918	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2919	{
2920	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2921	return VERR_VMX_INVALID_HOST_STATE;
2922	}
2923
2924	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2925	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2926	if (HMVMX_IS_64BIT_HOST_MODE())
2927	{
2928	/* We need the 64-bit TR base for hybrid darwin. */
2929	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2930	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2931	}
2932	else
2933	#endif
2934	{
2935	uintptr_t uTRBase;
2936	#if HC_ARCH_BITS == 64
2937	uTRBase = X86DESC64_BASE(pDesc);
2938
2939	/*
2940	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2941	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2942	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2943	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2944	*
2945	* [1] See Intel spec. 3.5 "System Descriptor Types".
2946	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2947	*/
2948	Assert(pDesc->System.u4Type == 11);
2949	if ( pDesc->System.u16LimitLow != 0x67
2950	\|\| pDesc->System.u4LimitHigh)
2951	{
2952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2953	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2954
2955	/* Store the GDTR here as we need it while restoring TR. */
2956	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2957	}
2958	#else
2959	uTRBase = X86DESC_BASE(pDesc);
2960	#endif
2961	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2962	}
2963	AssertRCReturn(rc, rc);
2964
2965	/*
2966	* Host FS base and GS base.
2967	*/
2968	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2969	if (HMVMX_IS_64BIT_HOST_MODE())
2970	{
2971	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2972	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2973	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2974	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2975
2976	# if HC_ARCH_BITS == 64
2977	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2978	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2979	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2980	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2981	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2982	# endif
2983	}
2984	#endif
2985	return rc;
2986	}
2987
2988
2989	/**
2990	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2991	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2992	* the host after every successful VM-exit.
2993	*
2994	* @returns VBox status code.
2995	* @param pVM Pointer to the VM.
2996	* @param pVCpu Pointer to the VMCPU.
2997	*
2998	* @remarks No-long-jump zone!!!
2999	*/
3000	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3001	{
3002	NOREF(pVM);
3003
3004	AssertPtr(pVCpu);
3005	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3006
3007	int rc = VINF_SUCCESS;
3008	#if HC_ARCH_BITS == 64
3009	if (pVM->hm.s.fAllow64BitGuests)
3010	hmR0VmxLazySaveHostMsrs(pVCpu);
3011	#endif
3012
3013	if (pVCpu->hm.s.vmx.cMsrs > 0)
3014	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3015
3016	/*
3017	* Host Sysenter MSRs.
3018	*/
3019	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3020	AssertRCReturn(rc, rc);
3021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3022	if (HMVMX_IS_64BIT_HOST_MODE())
3023	{
3024	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3025	AssertRCReturn(rc, rc);
3026	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3027	}
3028	else
3029	{
3030	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3031	AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3033	}
3034	#elif HC_ARCH_BITS == 32
3035	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3036	AssertRCReturn(rc, rc);
3037	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3038	#else
3039	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3042	#endif
3043	AssertRCReturn(rc, rc);
3044
3045	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3046	* hmR0VmxSetupExitCtls() !! */
3047	return rc;
3048	}
3049
3050
3051	/**
3052	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3053	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3054	* controls".
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu Pointer to the VMCPU.
3058	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3059	* out-of-sync. Make sure to update the required fields
3060	* before using them.
3061	*
3062	* @remarks No-long-jump zone!!!
3063	*/
3064	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3065	{
3066	int rc = VINF_SUCCESS;
3067	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3068	{
3069	PVM pVM = pVCpu->CTX_SUFF(pVM);
3070	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3071	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3072
3073	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3074	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3075
3076	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3077	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3078	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3079	else
3080	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3081
3082	/*
3083	* The following should -not- be set (since we're not in SMM mode):
3084	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3085	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3086	*/
3087
3088	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3089	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3090	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3091
3092	if ((val & zap) != val)
3093	{
3094	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3095	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3096	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3097	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3098	}
3099
3100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3101	AssertRCReturn(rc, rc);
3102
3103	/* Update VCPU with the currently set VM-exit controls. */
3104	pVCpu->hm.s.vmx.u32EntryCtls = val;
3105	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3106	}
3107	return rc;
3108	}
3109
3110
3111	/**
3112	* Sets up the VM-exit controls in the VMCS.
3113	*
3114	* @returns VBox status code.
3115	* @param pVM Pointer to the VM.
3116	* @param pVCpu Pointer to the VMCPU.
3117	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3118	* out-of-sync. Make sure to update the required fields
3119	* before using them.
3120	*
3121	* @remarks requires EFER.
3122	*/
3123	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3124	{
3125	NOREF(pMixedCtx);
3126
3127	int rc = VINF_SUCCESS;
3128	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3129	{
3130	PVM pVM = pVCpu->CTX_SUFF(pVM);
3131	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3135	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3136
3137	/*
3138	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3139	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3140	*/
3141	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3142	if (HMVMX_IS_64BIT_HOST_MODE())
3143	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3144	else
3145	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3146	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3147	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3148	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3149	else
3150	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3151	#endif
3152
3153	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3154	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3155
3156	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3157	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3158	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3159	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3160	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3161
3162	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3163	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3164
3165	if ((val & zap) != val)
3166	{
3167	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3168	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3169	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3170	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3171	}
3172
3173	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3174	AssertRCReturn(rc, rc);
3175
3176	/* Update VCPU with the currently set VM-exit controls. */
3177	pVCpu->hm.s.vmx.u32ExitCtls = val;
3178	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3179	}
3180	return rc;
3181	}
3182
3183
3184	/**
3185	* Loads the guest APIC and related state.
3186	*
3187	* @returns VBox status code.
3188	* @param pVM Pointer to the VM.
3189	* @param pVCpu Pointer to the VMCPU.
3190	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3191	* out-of-sync. Make sure to update the required fields
3192	* before using them.
3193	*/
3194	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3195	{
3196	NOREF(pMixedCtx);
3197
3198	int rc = VINF_SUCCESS;
3199	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3200	{
3201	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3202	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3203	{
3204	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3205
3206	bool fPendingIntr = false;
3207	uint8_t u8Tpr = 0;
3208	uint8_t u8PendingIntr = 0;
3209	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3210	AssertRCReturn(rc, rc);
3211
3212	/*
3213	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3214	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3215	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3216	* the interrupt when we VM-exit for other reasons.
3217	*/
3218	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3219	uint32_t u32TprThreshold = 0;
3220	if (fPendingIntr)
3221	{
3222	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3223	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3224	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3225	if (u8PendingPriority <= u8TprPriority)
3226	u32TprThreshold = u8PendingPriority;
3227	else
3228	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3229	}
3230	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3231
3232	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3233	AssertRCReturn(rc, rc);
3234	}
3235
3236	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3237	}
3238	return rc;
3239	}
3240
3241
3242	/**
3243	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3244	*
3245	* @returns Guest's interruptibility-state.
3246	* @param pVCpu Pointer to the VMCPU.
3247	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3248	* out-of-sync. Make sure to update the required fields
3249	* before using them.
3250	*
3251	* @remarks No-long-jump zone!!!
3252	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3253	*/
3254	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3255	{
3256	/*
3257	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3258	* inhibit interrupts or clear any existing interrupt-inhibition.
3259	*/
3260	uint32_t uIntrState = 0;
3261	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3262	{
3263	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3264	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3265	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3266	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3267	{
3268	/*
3269	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3270	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3271	*/
3272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3273	}
3274	else if (pMixedCtx->eflags.Bits.u1IF)
3275	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3276	else
3277	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3278	}
3279	return uIntrState;
3280	}
3281
3282
3283	/**
3284	* Loads the guest's interruptibility-state into the guest-state area in the
3285	* VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param uIntrState The interruptibility-state to set.
3290	*/
3291	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3292	{
3293	NOREF(pVCpu);
3294	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3295	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3296	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3297	AssertRCReturn(rc, rc);
3298	return rc;
3299	}
3300
3301
3302	/**
3303	* Loads the guest's RIP into the guest-state area in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu Pointer to the VMCPU.
3307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308	* out-of-sync. Make sure to update the required fields
3309	* before using them.
3310	*
3311	* @remarks No-long-jump zone!!!
3312	*/
3313	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3314	{
3315	int rc = VINF_SUCCESS;
3316	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3317	{
3318	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3319	AssertRCReturn(rc, rc);
3320
3321	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3322	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3323	}
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Loads the guest's RSP into the guest-state area in the VMCS.
3330	*
3331	* @returns VBox status code.
3332	* @param pVCpu Pointer to the VMCPU.
3333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3334	* out-of-sync. Make sure to update the required fields
3335	* before using them.
3336	*
3337	* @remarks No-long-jump zone!!!
3338	*/
3339	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3340	{
3341	int rc = VINF_SUCCESS;
3342	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3343	{
3344	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3345	AssertRCReturn(rc, rc);
3346
3347	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3348	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3349	}
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu Pointer to the VMCPU.
3359	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360	* out-of-sync. Make sure to update the required fields
3361	* before using them.
3362	*
3363	* @remarks No-long-jump zone!!!
3364	*/
3365	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366	{
3367	int rc = VINF_SUCCESS;
3368	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3369	{
3370	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3371	Let us assert it as such and use 32-bit VMWRITE. */
3372	Assert(!(pMixedCtx->rflags.u64 >> 32));
3373	X86EFLAGS Eflags = pMixedCtx->eflags;
3374	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3375	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3376
3377	/*
3378	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3379	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3380	*/
3381	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3382	{
3383	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3384	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3385	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3386	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3387	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3388	}
3389
3390	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3391	AssertRCReturn(rc, rc);
3392
3393	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3394	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3395	}
3396	return rc;
3397	}
3398
3399
3400	/**
3401	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3402	*
3403	* @returns VBox status code.
3404	* @param pVCpu Pointer to the VMCPU.
3405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3406	* out-of-sync. Make sure to update the required fields
3407	* before using them.
3408	*
3409	* @remarks No-long-jump zone!!!
3410	*/
3411	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3412	{
3413	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3414	AssertRCReturn(rc, rc);
3415	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3416	AssertRCReturn(rc, rc);
3417	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3418	AssertRCReturn(rc, rc);
3419	return rc;
3420	}
3421
3422
3423	/**
3424	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3425	* CR0 is partially shared with the host and we have to consider the FPU bits.
3426	*
3427	* @returns VBox status code.
3428	* @param pVM Pointer to the VM.
3429	* @param pVCpu Pointer to the VMCPU.
3430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3431	* out-of-sync. Make sure to update the required fields
3432	* before using them.
3433	*
3434	* @remarks No-long-jump zone!!!
3435	*/
3436	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437	{
3438	/*
3439	* Guest CR0.
3440	* Guest FPU.
3441	*/
3442	int rc = VINF_SUCCESS;
3443	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3444	{
3445	Assert(!(pMixedCtx->cr0 >> 32));
3446	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3447	PVM pVM = pVCpu->CTX_SUFF(pVM);
3448
3449	/* The guest's view (read access) of its CR0 is unblemished. */
3450	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3451	AssertRCReturn(rc, rc);
3452	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3453
3454	/* Setup VT-x's view of the guest CR0. */
3455	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3456	if (pVM->hm.s.fNestedPaging)
3457	{
3458	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3459	{
3460	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3461	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3462	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3463	}
3464	else
3465	{
3466	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3467	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3468	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3469	}
3470
3471	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3472	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3473	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3474
3475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3476	AssertRCReturn(rc, rc);
3477	}
3478	else
3479	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3480
3481	/*
3482	* Guest FPU bits.
3483	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3484	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3485	*/
3486	u32GuestCR0 \|= X86_CR0_NE;
3487	bool fInterceptNM = false;
3488	if (CPUMIsGuestFPUStateActive(pVCpu))
3489	{
3490	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3491	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3492	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3493	}
3494	else
3495	{
3496	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3497	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3498	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3499	}
3500
3501	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3502	bool fInterceptMF = false;
3503	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3504	fInterceptMF = true;
3505
3506	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3507	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3508	{
3509	Assert(PDMVmmDevHeapIsEnabled(pVM));
3510	Assert(pVM->hm.s.vmx.pRealModeTSS);
3511	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3512	fInterceptNM = true;
3513	fInterceptMF = true;
3514	}
3515	else
3516	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3517
3518	if (fInterceptNM)
3519	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3520	else
3521	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3522
3523	if (fInterceptMF)
3524	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3525	else
3526	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3527
3528	/* Additional intercepts for debugging, define these yourself explicitly. */
3529	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3531	\| RT_BIT(X86_XCPT_BP)
3532	\| RT_BIT(X86_XCPT_DB)
3533	\| RT_BIT(X86_XCPT_DE)
3534	\| RT_BIT(X86_XCPT_NM)
3535	\| RT_BIT(X86_XCPT_UD)
3536	\| RT_BIT(X86_XCPT_NP)
3537	\| RT_BIT(X86_XCPT_SS)
3538	\| RT_BIT(X86_XCPT_GP)
3539	\| RT_BIT(X86_XCPT_PF)
3540	\| RT_BIT(X86_XCPT_MF)
3541	;
3542	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3543	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3544	#endif
3545
3546	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3547
3548	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3549	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3550	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3551	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3552	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3553	else
3554	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3555
3556	u32GuestCR0 \|= uSetCR0;
3557	u32GuestCR0 &= uZapCR0;
3558	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3559
3560	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3561	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3562	AssertRCReturn(rc, rc);
3563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3564	AssertRCReturn(rc, rc);
3565	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3566
3567	/*
3568	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3569	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3570	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3571	*/
3572	uint32_t u32CR0Mask = 0;
3573	u32CR0Mask = X86_CR0_PE
3574	\| X86_CR0_NE
3575	\| X86_CR0_WP
3576	\| X86_CR0_PG
3577	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3578	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3579	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3580
3581	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3582	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3583	* and @bugref{6944}. */
3584	#if 0
3585	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3586	u32CR0Mask &= ~X86_CR0_PE;
3587	#endif
3588	if (pVM->hm.s.fNestedPaging)
3589	u32CR0Mask &= ~X86_CR0_WP;
3590
3591	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3592	if (fInterceptNM)
3593	{
3594	u32CR0Mask \|= X86_CR0_TS
3595	\| X86_CR0_MP;
3596	}
3597
3598	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3599	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3600	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3601	AssertRCReturn(rc, rc);
3602	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3603
3604	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3605	}
3606	return rc;
3607	}
3608
3609
3610	/**
3611	* Loads the guest control registers (CR3, CR4) into the guest-state area
3612	* in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVM Pointer to the VM.
3616	* @param pVCpu Pointer to the VMCPU.
3617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618	* out-of-sync. Make sure to update the required fields
3619	* before using them.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624	{
3625	int rc = VINF_SUCCESS;
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627
3628	/*
3629	* Guest CR2.
3630	* It's always loaded in the assembler code. Nothing to do here.
3631	*/
3632
3633	/*
3634	* Guest CR3.
3635	*/
3636	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3637	{
3638	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3639	if (pVM->hm.s.fNestedPaging)
3640	{
3641	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3642
3643	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3644	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3645	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3646	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3647
3648	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3649	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3650	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3651
3652	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3653	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3654	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3655	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3656
3657	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3658	AssertRCReturn(rc, rc);
3659	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3660
3661	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3662	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3663	{
3664	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3665	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3666	{
3667	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3668	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3669	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3670	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3671	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3672	}
3673
3674	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3675	have Unrestricted Execution to handle the guest when it's not using paging. */
3676	GCPhysGuestCR3 = pMixedCtx->cr3;
3677	}
3678	else
3679	{
3680	/*
3681	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3682	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3683	* EPT takes care of translating it to host-physical addresses.
3684	*/
3685	RTGCPHYS GCPhys;
3686	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3687	Assert(PDMVmmDevHeapIsEnabled(pVM));
3688
3689	/* We obtain it here every time as the guest could have relocated this PCI region. */
3690	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3691	AssertRCReturn(rc, rc);
3692
3693	GCPhysGuestCR3 = GCPhys;
3694	}
3695
3696	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3697	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3698	}
3699	else
3700	{
3701	/* Non-nested paging case, just use the hypervisor's CR3. */
3702	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3703
3704	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3705	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3706	}
3707	AssertRCReturn(rc, rc);
3708
3709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3710	}
3711
3712	/*
3713	* Guest CR4.
3714	*/
3715	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3716	{
3717	Assert(!(pMixedCtx->cr4 >> 32));
3718	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3719
3720	/* The guest's view of its CR4 is unblemished. */
3721	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3722	AssertRCReturn(rc, rc);
3723	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3724
3725	/* Setup VT-x's view of the guest CR4. */
3726	/*
3727	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3728	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3729	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3730	*/
3731	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3732	{
3733	Assert(pVM->hm.s.vmx.pRealModeTSS);
3734	Assert(PDMVmmDevHeapIsEnabled(pVM));
3735	u32GuestCR4 &= ~X86_CR4_VME;
3736	}
3737
3738	if (pVM->hm.s.fNestedPaging)
3739	{
3740	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3741	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3742	{
3743	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3744	u32GuestCR4 \|= X86_CR4_PSE;
3745	/* Our identity mapping is a 32-bit page directory. */
3746	u32GuestCR4 &= ~X86_CR4_PAE;
3747	}
3748	/* else use guest CR4.*/
3749	}
3750	else
3751	{
3752	/*
3753	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3754	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3755	*/
3756	switch (pVCpu->hm.s.enmShadowMode)
3757	{
3758	case PGMMODE_REAL: /* Real-mode. */
3759	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3760	case PGMMODE_32_BIT: /* 32-bit paging. */
3761	{
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	break;
3764	}
3765
3766	case PGMMODE_PAE: /* PAE paging. */
3767	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3768	{
3769	u32GuestCR4 \|= X86_CR4_PAE;
3770	break;
3771	}
3772
3773	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3774	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3775	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3776	break;
3777	#endif
3778	default:
3779	AssertFailed();
3780	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3781	}
3782	}
3783
3784	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3785	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3786	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3787	u32GuestCR4 \|= uSetCR4;
3788	u32GuestCR4 &= uZapCR4;
3789
3790	/* Write VT-x's view of the guest CR4 into the VMCS. */
3791	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3792	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3793	AssertRCReturn(rc, rc);
3794
3795	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3796	uint32_t u32CR4Mask = 0;
3797	u32CR4Mask = X86_CR4_VME
3798	\| X86_CR4_PAE
3799	\| X86_CR4_PGE
3800	\| X86_CR4_PSE
3801	\| X86_CR4_VMXE;
3802	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3803	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3804	AssertRCReturn(rc, rc);
3805
3806	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3807	}
3808	return rc;
3809	}
3810
3811
3812	/**
3813	* Loads the guest debug registers into the guest-state area in the VMCS.
3814	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3815	*
3816	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3817	*
3818	* @returns VBox status code.
3819	* @param pVCpu Pointer to the VMCPU.
3820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3821	* out-of-sync. Make sure to update the required fields
3822	* before using them.
3823	*
3824	* @remarks No-long-jump zone!!!
3825	*/
3826	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3827	{
3828	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3829	return VINF_SUCCESS;
3830
3831	#ifdef VBOX_STRICT
3832	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3833	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3834	{
3835	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3836	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3837	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3838	}
3839	#endif
3840
3841	int rc;
3842	PVM pVM = pVCpu->CTX_SUFF(pVM);
3843	bool fInterceptDB = false;
3844	bool fInterceptMovDRx = false;
3845	if ( pVCpu->hm.s.fSingleInstruction
3846	\|\| DBGFIsStepping(pVCpu))
3847	{
3848	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3849	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3850	{
3851	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3852	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3853	AssertRCReturn(rc, rc);
3854	Assert(fInterceptDB == false);
3855	}
3856	else
3857	{
3858	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3859	pVCpu->hm.s.fClearTrapFlag = true;
3860	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3861	fInterceptDB = true;
3862	}
3863	}
3864
3865	if ( fInterceptDB
3866	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3867	{
3868	/*
3869	* Use the combined guest and host DRx values found in the hypervisor
3870	* register set because the debugger has breakpoints active or someone
3871	* is single stepping on the host side without a monitor trap flag.
3872	*
3873	* Note! DBGF expects a clean DR6 state before executing guest code.
3874	*/
3875	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3876	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3877	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3878	{
3879	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3880	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3881	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3882	}
3883	else
3884	#endif
3885	if (!CPUMIsHyperDebugStateActive(pVCpu))
3886	{
3887	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3888	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3889	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3890	}
3891
3892	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3893	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3894	AssertRCReturn(rc, rc);
3895
3896	pVCpu->hm.s.fUsingHyperDR7 = true;
3897	fInterceptDB = true;
3898	fInterceptMovDRx = true;
3899	}
3900	else
3901	{
3902	/*
3903	* If the guest has enabled debug registers, we need to load them prior to
3904	* executing guest code so they'll trigger at the right time.
3905	*/
3906	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3907	{
3908	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3909	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3910	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3911	{
3912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3913	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3914	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3915	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3916	}
3917	else
3918	#endif
3919	if (!CPUMIsGuestDebugStateActive(pVCpu))
3920	{
3921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3922	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3924	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3925	}
3926	Assert(!fInterceptDB);
3927	Assert(!fInterceptMovDRx);
3928	}
3929	/*
3930	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3931	* must intercept #DB in order to maintain a correct DR6 guest value.
3932	*/
3933	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3935	&& !CPUMIsGuestDebugStateActive(pVCpu))
3936	#else
3937	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3938	#endif
3939	{
3940	fInterceptMovDRx = true;
3941	fInterceptDB = true;
3942	}
3943
3944	/* Update guest DR7. */
3945	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3946	AssertRCReturn(rc, rc);
3947
3948	pVCpu->hm.s.fUsingHyperDR7 = false;
3949	}
3950
3951	/*
3952	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3953	*/
3954	if (fInterceptDB)
3955	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3956	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3957	{
3958	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3959	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3960	#endif
3961	}
3962	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3963	AssertRCReturn(rc, rc);
3964
3965	/*
3966	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3967	*/
3968	if (fInterceptMovDRx)
3969	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3970	else
3971	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3973	AssertRCReturn(rc, rc);
3974
3975	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3976	return VINF_SUCCESS;
3977	}
3978
3979
3980	#ifdef VBOX_STRICT
3981	/**
3982	* Strict function to validate segment registers.
3983	*
3984	* @remarks ASSUMES CR0 is up to date.
3985	*/
3986	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3987	{
3988	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3989	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3990	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3991	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3992	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3993	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3994	{
3995	/* Protected mode checks */
3996	/* CS */
3997	Assert(pCtx->cs.Attr.n.u1Present);
3998	Assert(!(pCtx->cs.Attr.u & 0xf00));
3999	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4000	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4001	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4002	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4003	\|\| (pCtx->cs.Attr.n.u1Granularity));
4004	/* CS cannot be loaded with NULL in protected mode. */
4005	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4006	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4007	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4008	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4009	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4010	else
4011	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4012	/* SS */
4013	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4014	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4015	if ( !(pCtx->cr0 & X86_CR0_PE)
4016	\|\| pCtx->cs.Attr.n.u4Type == 3)
4017	{
4018	Assert(!pCtx->ss.Attr.n.u2Dpl);
4019	}
4020	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4021	{
4022	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4023	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4024	Assert(pCtx->ss.Attr.n.u1Present);
4025	Assert(!(pCtx->ss.Attr.u & 0xf00));
4026	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4027	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4028	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4029	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4030	\|\| (pCtx->ss.Attr.n.u1Granularity));
4031	}
4032	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4033	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4034	{
4035	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4036	Assert(pCtx->ds.Attr.n.u1Present);
4037	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4038	Assert(!(pCtx->ds.Attr.u & 0xf00));
4039	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4040	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4041	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4042	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4043	\|\| (pCtx->ds.Attr.n.u1Granularity));
4044	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4045	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4046	}
4047	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4048	{
4049	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4050	Assert(pCtx->es.Attr.n.u1Present);
4051	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4052	Assert(!(pCtx->es.Attr.u & 0xf00));
4053	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4054	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4055	\|\| !(pCtx->es.Attr.n.u1Granularity));
4056	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4057	\|\| (pCtx->es.Attr.n.u1Granularity));
4058	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4059	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4060	}
4061	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4062	{
4063	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4064	Assert(pCtx->fs.Attr.n.u1Present);
4065	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4066	Assert(!(pCtx->fs.Attr.u & 0xf00));
4067	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4068	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4069	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4070	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4071	\|\| (pCtx->fs.Attr.n.u1Granularity));
4072	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4073	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4074	}
4075	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4076	{
4077	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4078	Assert(pCtx->gs.Attr.n.u1Present);
4079	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4080	Assert(!(pCtx->gs.Attr.u & 0xf00));
4081	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4082	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4083	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4084	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4085	\|\| (pCtx->gs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4087	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4088	}
4089	/* 64-bit capable CPUs. */
4090	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4091	if (HMVMX_IS_64BIT_HOST_MODE())
4092	{
4093	Assert(!(pCtx->cs.u64Base >> 32));
4094	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4095	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4096	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4097	}
4098	# endif
4099	}
4100	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4101	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4102	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4103	{
4104	/* Real and v86 mode checks. */
4105	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4106	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4108	{
4109	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4110	}
4111	else
4112	{
4113	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4114	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4115	}
4116
4117	/* CS */
4118	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4119	Assert(pCtx->cs.u32Limit == 0xffff);
4120	Assert(u32CSAttr == 0xf3);
4121	/* SS */
4122	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4123	Assert(pCtx->ss.u32Limit == 0xffff);
4124	Assert(u32SSAttr == 0xf3);
4125	/* DS */
4126	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4127	Assert(pCtx->ds.u32Limit == 0xffff);
4128	Assert(u32DSAttr == 0xf3);
4129	/* ES */
4130	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4131	Assert(pCtx->es.u32Limit == 0xffff);
4132	Assert(u32ESAttr == 0xf3);
4133	/* FS */
4134	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4135	Assert(pCtx->fs.u32Limit == 0xffff);
4136	Assert(u32FSAttr == 0xf3);
4137	/* GS */
4138	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4139	Assert(pCtx->gs.u32Limit == 0xffff);
4140	Assert(u32GSAttr == 0xf3);
4141	/* 64-bit capable CPUs. */
4142	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143	if (HMVMX_IS_64BIT_HOST_MODE())
4144	{
4145	Assert(!(pCtx->cs.u64Base >> 32));
4146	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4147	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4148	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4149	}
4150	# endif
4151	}
4152	}
4153	#endif /* VBOX_STRICT */
4154
4155
4156	/**
4157	* Writes a guest segment register into the guest-state area in the VMCS.
4158	*
4159	* @returns VBox status code.
4160	* @param pVCpu Pointer to the VMCPU.
4161	* @param idxSel Index of the selector in the VMCS.
4162	* @param idxLimit Index of the segment limit in the VMCS.
4163	* @param idxBase Index of the segment base in the VMCS.
4164	* @param idxAccess Index of the access rights of the segment in the VMCS.
4165	* @param pSelReg Pointer to the segment selector.
4166	*
4167	* @remarks No-long-jump zone!!!
4168	*/
4169	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4170	uint32_t idxAccess, PCPUMSELREG pSelReg)
4171	{
4172	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4175	AssertRCReturn(rc, rc);
4176	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4177	AssertRCReturn(rc, rc);
4178
4179	uint32_t u32Access = pSelReg->Attr.u;
4180	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181	{
4182	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4183	u32Access = 0xf3;
4184	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4185	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4186	}
4187	else
4188	{
4189	/*
4190	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4191	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4192	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4193	* loaded in protected-mode have their attribute as 0.
4194	*/
4195	if (!u32Access)
4196	u32Access = X86DESCATTR_UNUSABLE;
4197	}
4198
4199	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4200	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4201	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4202
4203	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4204	AssertRCReturn(rc, rc);
4205	return rc;
4206	}
4207
4208
4209	/**
4210	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4211	* into the guest-state area in the VMCS.
4212	*
4213	* @returns VBox status code.
4214	* @param pVM Pointer to the VM.
4215	* @param pVCPU Pointer to the VMCPU.
4216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217	* out-of-sync. Make sure to update the required fields
4218	* before using them.
4219	*
4220	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4224	{
4225	int rc = VERR_INTERNAL_ERROR_5;
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/*
4229	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4230	*/
4231	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4232	{
4233	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4234	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4235	{
4236	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4240	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4242	}
4243
4244	#ifdef VBOX_WITH_REM
4245	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4246	{
4247	Assert(pVM->hm.s.vmx.pRealModeTSS);
4248	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4249	if ( pVCpu->hm.s.vmx.fWasInRealMode
4250	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4251	{
4252	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4253	in real-mode (e.g. OpenBSD 4.0) */
4254	REMFlushTBs(pVM);
4255	Log4(("Load: Switch to protected mode detected!\n"));
4256	pVCpu->hm.s.vmx.fWasInRealMode = false;
4257	}
4258	}
4259	#endif
4260	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4261	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4262	AssertRCReturn(rc, rc);
4263	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4264	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4265	AssertRCReturn(rc, rc);
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4267	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4270	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4273	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4276	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4277	AssertRCReturn(rc, rc);
4278
4279	#ifdef VBOX_STRICT
4280	/* Validate. */
4281	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4282	#endif
4283
4284	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4285	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4286	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4287	}
4288
4289	/*
4290	* Guest TR.
4291	*/
4292	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4293	{
4294	/*
4295	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4296	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4297	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4298	*/
4299	uint16_t u16Sel = 0;
4300	uint32_t u32Limit = 0;
4301	uint64_t u64Base = 0;
4302	uint32_t u32AccessRights = 0;
4303
4304	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4305	{
4306	u16Sel = pMixedCtx->tr.Sel;
4307	u32Limit = pMixedCtx->tr.u32Limit;
4308	u64Base = pMixedCtx->tr.u64Base;
4309	u32AccessRights = pMixedCtx->tr.Attr.u;
4310	}
4311	else
4312	{
4313	Assert(pVM->hm.s.vmx.pRealModeTSS);
4314	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4315
4316	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4317	RTGCPHYS GCPhys;
4318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4319	AssertRCReturn(rc, rc);
4320
4321	X86DESCATTR DescAttr;
4322	DescAttr.u = 0;
4323	DescAttr.n.u1Present = 1;
4324	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4325
4326	u16Sel = 0;
4327	u32Limit = HM_VTX_TSS_SIZE;
4328	u64Base = GCPhys; /* in real-mode phys = virt. */
4329	u32AccessRights = DescAttr.u;
4330	}
4331
4332	/* Validate. */
4333	Assert(!(u16Sel & RT_BIT(2)));
4334	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4335	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4336	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4337	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4338	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4339	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4340	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4341	Assert( (u32Limit & 0xfff) == 0xfff
4342	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4343	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4344	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4345
4346	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4349	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4350
4351	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4352	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4353	}
4354
4355	/*
4356	* Guest GDTR.
4357	*/
4358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4359	{
4360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4361	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4362
4363	/* Validate. */
4364	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4365
4366	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4367	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4368	}
4369
4370	/*
4371	* Guest LDTR.
4372	*/
4373	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4374	{
4375	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4376	uint32_t u32Access = 0;
4377	if (!pMixedCtx->ldtr.Attr.u)
4378	u32Access = X86DESCATTR_UNUSABLE;
4379	else
4380	u32Access = pMixedCtx->ldtr.Attr.u;
4381
4382	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4384	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4385	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4386
4387	/* Validate. */
4388	if (!(u32Access & X86DESCATTR_UNUSABLE))
4389	{
4390	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4393	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4394	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4395	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4396	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4397	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4398	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4399	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4400	}
4401
4402	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4403	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4404	}
4405
4406	/*
4407	* Guest IDTR.
4408	*/
4409	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4410	{
4411	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4412	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4413
4414	/* Validate. */
4415	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4416
4417	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4419	}
4420
4421	return VINF_SUCCESS;
4422	}
4423
4424
4425	/**
4426	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4427	* areas. These MSRs will automatically be loaded to the host CPU on every
4428	* successful VM entry and stored from the host CPU on every successful VM-exit.
4429	*
4430	* This also creates/updates MSR slots for the host MSRs. The actual host
4431	* MSR values are -not- updated here for performance reasons. See
4432	* hmR0VmxSaveHostMsrs().
4433	*
4434	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4445	{
4446	AssertPtr(pVCpu);
4447	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4448
4449	/*
4450	* MSRs that we use the auto-load/store MSR area in the VMCS.
4451	*/
4452	PVM pVM = pVCpu->CTX_SUFF(pVM);
4453	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4454	{
4455	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4456	if (pVM->hm.s.fAllow64BitGuests)
4457	{
4458	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4459	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4462	# ifdef DEBUG
4463	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4464	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4465	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4466	# endif
4467	}
4468	#endif
4469	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4470	}
4471
4472	/*
4473	* Guest Sysenter MSRs.
4474	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4475	* VM-exits on WRMSRs for these MSRs.
4476	*/
4477	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4478	{
4479	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4480	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4481	}
4482
4483	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4484	{
4485	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4486	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4487	}
4488
4489	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4490	{
4491	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4492	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4493	}
4494
4495	return VINF_SUCCESS;
4496	}
4497
4498
4499	/**
4500	* Loads the guest activity state into the guest-state area in the VMCS.
4501	*
4502	* @returns VBox status code.
4503	* @param pVCpu Pointer to the VMCPU.
4504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4505	* out-of-sync. Make sure to update the required fields
4506	* before using them.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4511	{
4512	NOREF(pCtx);
4513	/** @todo See if we can make use of other states, e.g.
4514	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4515	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4516	{
4517	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4518	AssertRCReturn(rc, rc);
4519
4520	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4521	}
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Sets up the appropriate function to run guest code.
4528	*
4529	* @returns VBox status code.
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4532	* out-of-sync. Make sure to update the required fields
4533	* before using them.
4534	*
4535	* @remarks No-long-jump zone!!!
4536	*/
4537	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4538	{
4539	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4540	{
4541	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4542	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4543	#endif
4544	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4545	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4546	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4547	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4548	{
4549	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4550	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4551	}
4552	#else
4553	/* 64-bit host or hybrid host. */
4554	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4555	#endif
4556	}
4557	else
4558	{
4559	/* Guest is not in long mode, use the 32-bit handler. */
4560	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4561	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4562	{
4563	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4564	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4565	}
4566	#else
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4568	#endif
4569	}
4570	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4571	return VINF_SUCCESS;
4572	}
4573
4574
4575	/**
4576	* Wrapper for running the guest code in VT-x.
4577	*
4578	* @returns VBox strict status code.
4579	* @param pVM Pointer to the VM.
4580	* @param pVCpu Pointer to the VMCPU.
4581	* @param pCtx Pointer to the guest-CPU context.
4582	*
4583	* @remarks No-long-jump zone!!!
4584	*/
4585	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4586	{
4587	/*
4588	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4589	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4590	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4591	*/
4592	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4593	/** @todo Add stats for resume vs launch. */
4594	#ifdef VBOX_WITH_KERNEL_USING_XMM
4595	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4596	#else
4597	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4598	#endif
4599	}
4600
4601
4602	/**
4603	* Reports world-switch error and dumps some useful debug info.
4604	*
4605	* @param pVM Pointer to the VM.
4606	* @param pVCpu Pointer to the VMCPU.
4607	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4608	* @param pCtx Pointer to the guest-CPU context.
4609	* @param pVmxTransient Pointer to the VMX transient structure (only
4610	* exitReason updated).
4611	*/
4612	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4613	{
4614	Assert(pVM);
4615	Assert(pVCpu);
4616	Assert(pCtx);
4617	Assert(pVmxTransient);
4618	HMVMX_ASSERT_PREEMPT_SAFE();
4619
4620	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4621	switch (rcVMRun)
4622	{
4623	case VERR_VMX_INVALID_VMXON_PTR:
4624	AssertFailed();
4625	break;
4626	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4627	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4628	{
4629	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4630	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4631	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4632	AssertRC(rc);
4633
4634	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4635	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4636	Cannot do it here as we may have been long preempted. */
4637
4638	#ifdef VBOX_STRICT
4639	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4640	pVmxTransient->uExitReason));
4641	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4642	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4643	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4644	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4645	else
4646	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4647	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4648	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4649
4650	/* VMX control bits. */
4651	uint32_t u32Val;
4652	uint64_t u64Val;
4653	HMVMXHCUINTREG uHCReg;
4654	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4655	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4656	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4657	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4659	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4686	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4687	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4688	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4689	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4691	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4694	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4695	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4696
4697	/* Guest bits. */
4698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4699	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4700	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4701	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4702	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4703	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4704	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4705	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4706
4707	/* Host bits. */
4708	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4709	Log4(("Host CR0 %#RHr\n", uHCReg));
4710	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4711	Log4(("Host CR3 %#RHr\n", uHCReg));
4712	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4713	Log4(("Host CR4 %#RHr\n", uHCReg));
4714
4715	RTGDTR HostGdtr;
4716	PCX86DESCHC pDesc;
4717	ASMGetGDTR(&HostGdtr);
4718	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4719	Log4(("Host CS %#08x\n", u32Val));
4720	if (u32Val < HostGdtr.cbGdt)
4721	{
4722	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4723	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4724	}
4725
4726	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4727	Log4(("Host DS %#08x\n", u32Val));
4728	if (u32Val < HostGdtr.cbGdt)
4729	{
4730	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4731	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4732	}
4733
4734	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4735	Log4(("Host ES %#08x\n", u32Val));
4736	if (u32Val < HostGdtr.cbGdt)
4737	{
4738	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4739	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4740	}
4741
4742	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4743	Log4(("Host FS %#08x\n", u32Val));
4744	if (u32Val < HostGdtr.cbGdt)
4745	{
4746	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4747	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4748	}
4749
4750	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4751	Log4(("Host GS %#08x\n", u32Val));
4752	if (u32Val < HostGdtr.cbGdt)
4753	{
4754	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4755	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4756	}
4757
4758	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4759	Log4(("Host SS %#08x\n", u32Val));
4760	if (u32Val < HostGdtr.cbGdt)
4761	{
4762	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4763	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4764	}
4765
4766	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4767	Log4(("Host TR %#08x\n", u32Val));
4768	if (u32Val < HostGdtr.cbGdt)
4769	{
4770	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4771	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4772	}
4773
4774	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4775	Log4(("Host TR Base %#RHv\n", uHCReg));
4776	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4777	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4778	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4779	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4780	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4781	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4783	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4784	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4785	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4786	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4787	Log4(("Host RSP %#RHv\n", uHCReg));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4789	Log4(("Host RIP %#RHv\n", uHCReg));
4790	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4791	if (HMVMX_IS_64BIT_HOST_MODE())
4792	{
4793	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4794	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4795	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4796	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4797	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4798	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4799	}
4800	# endif
4801	#endif /* VBOX_STRICT */
4802	break;
4803	}
4804
4805	default:
4806	/* Impossible */
4807	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4808	break;
4809	}
4810	NOREF(pVM); NOREF(pCtx);
4811	}
4812
4813
4814	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4815	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4816	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4817	#endif
4818	#ifdef VBOX_STRICT
4819	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4820	{
4821	switch (idxField)
4822	{
4823	case VMX_VMCS_GUEST_RIP:
4824	case VMX_VMCS_GUEST_RSP:
4825	case VMX_VMCS_GUEST_SYSENTER_EIP:
4826	case VMX_VMCS_GUEST_SYSENTER_ESP:
4827	case VMX_VMCS_GUEST_GDTR_BASE:
4828	case VMX_VMCS_GUEST_IDTR_BASE:
4829	case VMX_VMCS_GUEST_CS_BASE:
4830	case VMX_VMCS_GUEST_DS_BASE:
4831	case VMX_VMCS_GUEST_ES_BASE:
4832	case VMX_VMCS_GUEST_FS_BASE:
4833	case VMX_VMCS_GUEST_GS_BASE:
4834	case VMX_VMCS_GUEST_SS_BASE:
4835	case VMX_VMCS_GUEST_LDTR_BASE:
4836	case VMX_VMCS_GUEST_TR_BASE:
4837	case VMX_VMCS_GUEST_CR3:
4838	return true;
4839	}
4840	return false;
4841	}
4842
4843	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4844	{
4845	switch (idxField)
4846	{
4847	/* Read-only fields. */
4848	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4849	return true;
4850	}
4851	/* Remaining readable fields should also be writable. */
4852	return hmR0VmxIsValidWriteField(idxField);
4853	}
4854	#endif /* VBOX_STRICT */
4855
4856
4857	/**
4858	* Executes the specified handler in 64-bit mode.
4859	*
4860	* @returns VBox status code.
4861	* @param pVM Pointer to the VM.
4862	* @param pVCpu Pointer to the VMCPU.
4863	* @param pCtx Pointer to the guest CPU context.
4864	* @param enmOp The operation to perform.
4865	* @param cbParam Number of parameters.
4866	* @param paParam Array of 32-bit parameters.
4867	*/
4868	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4869	uint32_t *paParam)
4870	{
4871	int rc, rc2;
4872	PHMGLOBALCPUINFO pCpu;
4873	RTHCPHYS HCPhysCpuPage;
4874	RTCCUINTREG uOldEflags;
4875
4876	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4877	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4878	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4879	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4880
4881	#ifdef VBOX_STRICT
4882	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4883	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4884
4885	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4886	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4887	#endif
4888
4889	/* Disable interrupts. */
4890	uOldEflags = ASMIntDisableFlags();
4891
4892	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4893	RTCPUID idHostCpu = RTMpCpuId();
4894	CPUMR0SetLApic(pVCpu, idHostCpu);
4895	#endif
4896
4897	pCpu = HMR0GetCurrentCpu();
4898	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4899
4900	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4901	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4902
4903	/* Leave VMX Root Mode. */
4904	VMXDisable();
4905
4906	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4907
4908	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4909	CPUMSetHyperEIP(pVCpu, enmOp);
4910	for (int i = (int)cbParam - 1; i >= 0; i--)
4911	CPUMPushHyper(pVCpu, paParam[i]);
4912
4913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4914
4915	/* Call the switcher. */
4916	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4917	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4918
4919	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4920	/* Make sure the VMX instructions don't cause #UD faults. */
4921	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4922
4923	/* Re-enter VMX Root Mode */
4924	rc2 = VMXEnable(HCPhysCpuPage);
4925	if (RT_FAILURE(rc2))
4926	{
4927	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4928	ASMSetFlags(uOldEflags);
4929	return rc2;
4930	}
4931
4932	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4933	AssertRC(rc2);
4934	Assert(!(ASMGetFlags() & X86_EFL_IF));
4935	ASMSetFlags(uOldEflags);
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4942	* supporting 64-bit guests.
4943	*
4944	* @returns VBox status code.
4945	* @param fResume Whether to VMLAUNCH or VMRESUME.
4946	* @param pCtx Pointer to the guest-CPU context.
4947	* @param pCache Pointer to the VMCS cache.
4948	* @param pVM Pointer to the VM.
4949	* @param pVCpu Pointer to the VMCPU.
4950	*/
4951	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4952	{
4953	uint32_t aParam[6];
4954	PHMGLOBALCPUINFO pCpu = NULL;
4955	RTHCPHYS HCPhysCpuPage = 0;
4956	int rc = VERR_INTERNAL_ERROR_5;
4957
4958	pCpu = HMR0GetCurrentCpu();
4959	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4960
4961	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4962	pCache->uPos = 1;
4963	pCache->interPD = PGMGetInterPaeCR3(pVM);
4964	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4965	#endif
4966
4967	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4968	pCache->TestIn.HCPhysCpuPage = 0;
4969	pCache->TestIn.HCPhysVmcs = 0;
4970	pCache->TestIn.pCache = 0;
4971	pCache->TestOut.HCPhysVmcs = 0;
4972	pCache->TestOut.pCache = 0;
4973	pCache->TestOut.pCtx = 0;
4974	pCache->TestOut.eflags = 0;
4975	#endif
4976
4977	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4978	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4979	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4980	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4981	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4982	aParam[5] = 0;
4983
4984	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4985	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4986	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4987	#endif
4988	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4989
4990	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4991	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4992	Assert(pCtx->dr[4] == 10);
4993	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4994	#endif
4995
4996	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4997	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4998	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4999	pVCpu->hm.s.vmx.HCPhysVmcs));
5000	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5001	pCache->TestOut.HCPhysVmcs));
5002	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5003	pCache->TestOut.pCache));
5004	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5005	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5006	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5007	pCache->TestOut.pCtx));
5008	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5009	#endif
5010	return rc;
5011	}
5012
5013
5014	/**
5015	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5016	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5017	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5018	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5019	*
5020	* @returns VBox status code.
5021	* @param pVM Pointer to the VM.
5022	* @param pVCpu Pointer to the VMCPU.
5023	*/
5024	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5025	{
5026	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5027	{ \
5028	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5029	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5030	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5031	++cReadFields; \
5032	}
5033
5034	AssertPtr(pVM);
5035	AssertPtr(pVCpu);
5036	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5037	uint32_t cReadFields = 0;
5038
5039	/*
5040	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5041	* and serve to indicate exceptions to the rules.
5042	*/
5043
5044	/* Guest-natural selector base fields. */
5045	#if 0
5046	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5047	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5048	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5049	#endif
5050	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5062	#if 0
5063	/* Unused natural width guest-state fields. */
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5066	#endif
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5069
5070	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5071	#if 0
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5081	#endif
5082
5083	/* Natural width guest-state fields. */
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5085	#if 0
5086	/* Currently unused field. */
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5088	#endif
5089
5090	if (pVM->hm.s.fNestedPaging)
5091	{
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5093	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5094	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5095	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5096	}
5097	else
5098	{
5099	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5100	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5101	}
5102
5103	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5104	return VINF_SUCCESS;
5105	}
5106
5107
5108	/**
5109	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5110	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5111	* darwin, running 64-bit guests).
5112	*
5113	* @returns VBox status code.
5114	* @param pVCpu Pointer to the VMCPU.
5115	* @param idxField The VMCS field encoding.
5116	* @param u64Val 16, 32 or 64-bit value.
5117	*/
5118	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5119	{
5120	int rc;
5121	switch (idxField)
5122	{
5123	/*
5124	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5125	*/
5126	/* 64-bit Control fields. */
5127	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5128	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5129	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5130	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5131	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5132	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5133	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5134	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5135	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5136	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5137	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5138	case VMX_VMCS64_CTRL_EPTP_FULL:
5139	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5140	/* 64-bit Guest-state fields. */
5141	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5142	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5143	case VMX_VMCS64_GUEST_PAT_FULL:
5144	case VMX_VMCS64_GUEST_EFER_FULL:
5145	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5146	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5147	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5148	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5150	/* 64-bit Host-state fields. */
5151	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5152	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5153	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5154	{
5155	rc = VMXWriteVmcs32(idxField, u64Val);
5156	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5157	break;
5158	}
5159
5160	/*
5161	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5162	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5163	*/
5164	/* Natural-width Guest-state fields. */
5165	case VMX_VMCS_GUEST_CR3:
5166	case VMX_VMCS_GUEST_ES_BASE:
5167	case VMX_VMCS_GUEST_CS_BASE:
5168	case VMX_VMCS_GUEST_SS_BASE:
5169	case VMX_VMCS_GUEST_DS_BASE:
5170	case VMX_VMCS_GUEST_FS_BASE:
5171	case VMX_VMCS_GUEST_GS_BASE:
5172	case VMX_VMCS_GUEST_LDTR_BASE:
5173	case VMX_VMCS_GUEST_TR_BASE:
5174	case VMX_VMCS_GUEST_GDTR_BASE:
5175	case VMX_VMCS_GUEST_IDTR_BASE:
5176	case VMX_VMCS_GUEST_RSP:
5177	case VMX_VMCS_GUEST_RIP:
5178	case VMX_VMCS_GUEST_SYSENTER_ESP:
5179	case VMX_VMCS_GUEST_SYSENTER_EIP:
5180	{
5181	if (!(u64Val >> 32))
5182	{
5183	/* If this field is 64-bit, VT-x will zero out the top bits. */
5184	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5185	}
5186	else
5187	{
5188	/* Assert that only the 32->64 switcher case should ever come here. */
5189	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5190	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5191	}
5192	break;
5193	}
5194
5195	default:
5196	{
5197	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5198	rc = VERR_INVALID_PARAMETER;
5199	break;
5200	}
5201	}
5202	AssertRCReturn(rc, rc);
5203	return rc;
5204	}
5205
5206
5207	/**
5208	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5209	* hosts (except darwin) for 64-bit guests.
5210	*
5211	* @param pVCpu Pointer to the VMCPU.
5212	* @param idxField The VMCS field encoding.
5213	* @param u64Val 16, 32 or 64-bit value.
5214	*/
5215	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5216	{
5217	AssertPtr(pVCpu);
5218	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5219
5220	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5221	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5222
5223	/* Make sure there are no duplicates. */
5224	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5225	{
5226	if (pCache->Write.aField[i] == idxField)
5227	{
5228	pCache->Write.aFieldVal[i] = u64Val;
5229	return VINF_SUCCESS;
5230	}
5231	}
5232
5233	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5234	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5235	pCache->Write.cValidEntries++;
5236	return VINF_SUCCESS;
5237	}
5238
5239	/* Enable later when the assembly code uses these as callbacks. */
5240	#if 0
5241	/*
5242	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5243	*
5244	* @param pVCpu Pointer to the VMCPU.
5245	* @param pCache Pointer to the VMCS cache.
5246	*
5247	* @remarks No-long-jump zone!!!
5248	*/
5249	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5250	{
5251	AssertPtr(pCache);
5252	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5253	{
5254	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5255	AssertRC(rc);
5256	}
5257	pCache->Write.cValidEntries = 0;
5258	}
5259
5260
5261	/**
5262	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5263	*
5264	* @param pVCpu Pointer to the VMCPU.
5265	* @param pCache Pointer to the VMCS cache.
5266	*
5267	* @remarks No-long-jump zone!!!
5268	*/
5269	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5270	{
5271	AssertPtr(pCache);
5272	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5273	{
5274	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5275	AssertRC(rc);
5276	}
5277	}
5278	#endif
5279	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5280
5281
5282	/**
5283	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5284	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5285	* timer.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	*
5290	* @remarks No-long-jump zone!!!
5291	*/
5292	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5293	{
5294	int rc = VERR_INTERNAL_ERROR_5;
5295	bool fOffsettedTsc = false;
5296	PVM pVM = pVCpu->CTX_SUFF(pVM);
5297	if (pVM->hm.s.vmx.fUsePreemptTimer)
5298	{
5299	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5300
5301	/* Make sure the returned values have sane upper and lower boundaries. */
5302	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5303	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5304	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5305	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5306
5307	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5308	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5309	}
5310	else
5311	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5312
5313	if (fOffsettedTsc)
5314	{
5315	uint64_t u64CurTSC = ASMReadTSC();
5316	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5317	{
5318	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5319	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5320
5321	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5322	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5323	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5324	}
5325	else
5326	{
5327	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5329	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5331	}
5332	}
5333	else
5334	{
5335	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5336	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5339	}
5340	}
5341
5342
5343	/**
5344	* Determines if an exception is a contributory exception. Contributory
5345	* exceptions are ones which can cause double-faults. Page-fault is
5346	* intentionally not included here as it's a conditional contributory exception.
5347	*
5348	* @returns true if the exception is contributory, false otherwise.
5349	* @param uVector The exception vector.
5350	*/
5351	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5352	{
5353	switch (uVector)
5354	{
5355	case X86_XCPT_GP:
5356	case X86_XCPT_SS:
5357	case X86_XCPT_NP:
5358	case X86_XCPT_TS:
5359	case X86_XCPT_DE:
5360	return true;
5361	default:
5362	break;
5363	}
5364	return false;
5365	}
5366
5367
5368	/**
5369	* Sets an event as a pending event to be injected into the guest.
5370	*
5371	* @param pVCpu Pointer to the VMCPU.
5372	* @param u32IntInfo The VM-entry interruption-information field.
5373	* @param cbInstr The VM-entry instruction length in bytes (for software
5374	* interrupts, exceptions and privileged software
5375	* exceptions).
5376	* @param u32ErrCode The VM-entry exception error code.
5377	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5378	* page-fault.
5379	*
5380	* @remarks Statistics counter assumes this is a guest event being injected or
5381	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5382	* always incremented.
5383	*/
5384	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5385	RTGCUINTPTR GCPtrFaultAddress)
5386	{
5387	Assert(!pVCpu->hm.s.Event.fPending);
5388	pVCpu->hm.s.Event.fPending = true;
5389	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5390	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5391	pVCpu->hm.s.Event.cbInstr = cbInstr;
5392	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5393
5394	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5395	}
5396
5397
5398	/**
5399	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5400	*
5401	* @param pVCpu Pointer to the VMCPU.
5402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5403	* out-of-sync. Make sure to update the required fields
5404	* before using them.
5405	*/
5406	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5407	{
5408	NOREF(pMixedCtx);
5409	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5410	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5411	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5412	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5413	}
5414
5415
5416	/**
5417	* Handle a condition that occurred while delivering an event through the guest
5418	* IDT.
5419	*
5420	* @returns VBox status code (informational error codes included).
5421	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5422	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5423	* continue execution of the guest which will delivery the #DF.
5424	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5425	*
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	* @param pVmxTransient Pointer to the VMX transient structure.
5431	*
5432	* @remarks No-long-jump zone!!!
5433	*/
5434	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5435	{
5436	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5437	AssertRCReturn(rc, rc);
5438	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5439	{
5440	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5441	AssertRCReturn(rc, rc);
5442
5443	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5444	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5445	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5446
5447	typedef enum
5448	{
5449	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5450	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5451	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5452	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5453	} VMXREFLECTXCPT;
5454
5455	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5456	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5457	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5458	{
5459	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5460	{
5461	enmReflect = VMXREFLECTXCPT_XCPT;
5462	#ifdef VBOX_STRICT
5463	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5464	&& uExitVector == X86_XCPT_PF)
5465	{
5466	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5467	}
5468	#endif
5469	if ( uExitVector == X86_XCPT_PF
5470	&& uIdtVector == X86_XCPT_PF)
5471	{
5472	pVmxTransient->fVectoringPF = true;
5473	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5474	}
5475	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5476	&& hmR0VmxIsContributoryXcpt(uExitVector)
5477	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5478	\|\| uIdtVector == X86_XCPT_PF))
5479	{
5480	enmReflect = VMXREFLECTXCPT_DF;
5481	}
5482	else if (uIdtVector == X86_XCPT_DF)
5483	enmReflect = VMXREFLECTXCPT_TF;
5484	}
5485	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5486	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5487	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5488	{
5489	/*
5490	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5491	* (whatever they are) as they reoccur when restarting the instruction.
5492	*/
5493	enmReflect = VMXREFLECTXCPT_XCPT;
5494	}
5495	}
5496	else
5497	{
5498	/*
5499	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5500	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5501	* original exception to the guest after handling the VM-exit.
5502	*/
5503	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5504	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5506	{
5507	enmReflect = VMXREFLECTXCPT_XCPT;
5508	}
5509	}
5510
5511	switch (enmReflect)
5512	{
5513	case VMXREFLECTXCPT_XCPT:
5514	{
5515	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5516	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5517	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5518
5519	uint32_t u32ErrCode = 0;
5520	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5521	{
5522	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5523	AssertRCReturn(rc, rc);
5524	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5525	}
5526
5527	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5528	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5529	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5530	rc = VINF_SUCCESS;
5531	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5532	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5533
5534	break;
5535	}
5536
5537	case VMXREFLECTXCPT_DF:
5538	{
5539	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5540	rc = VINF_HM_DOUBLE_FAULT;
5541	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5542	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5543
5544	break;
5545	}
5546
5547	case VMXREFLECTXCPT_TF:
5548	{
5549	rc = VINF_EM_RESET;
5550	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5551	uExitVector));
5552	break;
5553	}
5554
5555	default:
5556	Assert(rc == VINF_SUCCESS);
5557	break;
5558	}
5559	}
5560	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5561	return rc;
5562	}
5563
5564
5565	/**
5566	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5567	*
5568	* @returns VBox status code.
5569	* @param pVCpu Pointer to the VMCPU.
5570	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5571	* out-of-sync. Make sure to update the required fields
5572	* before using them.
5573	*
5574	* @remarks No-long-jump zone!!!
5575	*/
5576	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5577	{
5578	NOREF(pMixedCtx);
5579
5580	/* Since this can be called from our preemption hook it's safer to make the guest-CR0 update non-preemptible. */
5581	VMMRZCallRing3Disable(pVCpu);
5582	HM_DISABLE_PREEMPT_IF_NEEDED();
5583
5584	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5585	{
5586	uint32_t uVal = 0;
5587	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5588	AssertRCReturn(rc, rc);
5589
5590	uint32_t uShadow = 0;
5591	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5592	AssertRCReturn(rc, rc);
5593
5594	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5595	CPUMSetGuestCR0(pVCpu, uVal);
5596	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5597	}
5598
5599	HM_RESTORE_PREEMPT_IF_NEEDED();
5600	VMMRZCallRing3Enable(pVCpu);
5601
5602	return VINF_SUCCESS;
5603	}
5604
5605
5606	/**
5607	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5608	*
5609	* @returns VBox status code.
5610	* @param pVCpu Pointer to the VMCPU.
5611	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5612	* out-of-sync. Make sure to update the required fields
5613	* before using them.
5614	*
5615	* @remarks No-long-jump zone!!!
5616	*/
5617	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5618	{
5619	NOREF(pMixedCtx);
5620
5621	int rc = VINF_SUCCESS;
5622	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5623	{
5624	uint32_t uVal = 0;
5625	uint32_t uShadow = 0;
5626	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5627	AssertRCReturn(rc, rc);
5628	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5629	AssertRCReturn(rc, rc);
5630
5631	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5632	CPUMSetGuestCR4(pVCpu, uVal);
5633	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5634	}
5635	return rc;
5636	}
5637
5638
5639	/**
5640	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5641	*
5642	* @returns VBox status code.
5643	* @param pVCpu Pointer to the VMCPU.
5644	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5645	* out-of-sync. Make sure to update the required fields
5646	* before using them.
5647	*
5648	* @remarks No-long-jump zone!!!
5649	*/
5650	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5651	{
5652	int rc = VINF_SUCCESS;
5653	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5654	{
5655	uint64_t u64Val = 0;
5656	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5657	AssertRCReturn(rc, rc);
5658
5659	pMixedCtx->rip = u64Val;
5660	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5661	}
5662	return rc;
5663	}
5664
5665
5666	/**
5667	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5668	*
5669	* @returns VBox status code.
5670	* @param pVCpu Pointer to the VMCPU.
5671	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5672	* out-of-sync. Make sure to update the required fields
5673	* before using them.
5674	*
5675	* @remarks No-long-jump zone!!!
5676	*/
5677	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5678	{
5679	int rc = VINF_SUCCESS;
5680	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5681	{
5682	uint64_t u64Val = 0;
5683	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5684	AssertRCReturn(rc, rc);
5685
5686	pMixedCtx->rsp = u64Val;
5687	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5688	}
5689	return rc;
5690	}
5691
5692
5693	/**
5694	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5695	*
5696	* @returns VBox status code.
5697	* @param pVCpu Pointer to the VMCPU.
5698	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5699	* out-of-sync. Make sure to update the required fields
5700	* before using them.
5701	*
5702	* @remarks No-long-jump zone!!!
5703	*/
5704	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5707	{
5708	uint32_t uVal = 0;
5709	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5710	AssertRCReturn(rc, rc);
5711
5712	pMixedCtx->eflags.u32 = uVal;
5713	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5714	{
5715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5716	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5717
5718	pMixedCtx->eflags.Bits.u1VM = 0;
5719	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5720	}
5721
5722	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5723	}
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	/**
5729	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5730	* guest-CPU context.
5731	*/
5732	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5733	{
5734	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5735	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5736	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5737	return rc;
5738	}
5739
5740
5741	/**
5742	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5743	* from the guest-state area in the VMCS.
5744	*
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	uint32_t uIntrState = 0;
5755	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5756	AssertRC(rc);
5757
5758	if (!uIntrState)
5759	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5760	else
5761	{
5762	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5763	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5764	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5765	AssertRC(rc);
5766	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5767	AssertRC(rc);
5768
5769	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5770	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5771	}
5772	}
5773
5774
5775	/**
5776	* Saves the guest's activity state.
5777	*
5778	* @returns VBox status code.
5779	* @param pVCpu Pointer to the VMCPU.
5780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781	* out-of-sync. Make sure to update the required fields
5782	* before using them.
5783	*
5784	* @remarks No-long-jump zone!!!
5785	*/
5786	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787	{
5788	NOREF(pMixedCtx);
5789	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5790	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5791	return VINF_SUCCESS;
5792	}
5793
5794
5795	/**
5796	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5797	* the current VMCS into the guest-CPU context.
5798	*
5799	* @returns VBox status code.
5800	* @param pVCpu Pointer to the VMCPU.
5801	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5802	* out-of-sync. Make sure to update the required fields
5803	* before using them.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5808	{
5809	int rc = VINF_SUCCESS;
5810	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5811	{
5812	uint32_t u32Val = 0;
5813	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5814	pMixedCtx->SysEnter.cs = u32Val;
5815	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5816	}
5817
5818	uint64_t u64Val = 0;
5819	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5822	pMixedCtx->SysEnter.eip = u64Val;
5823	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5824	}
5825	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5826	{
5827	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5828	pMixedCtx->SysEnter.esp = u64Val;
5829	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5830	}
5831	return rc;
5832	}
5833
5834
5835	/**
5836	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5837	* the CPU back into the guest-CPU context.
5838	*
5839	* @returns VBox status code.
5840	* @param pVCpu Pointer to the VMCPU.
5841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5842	* out-of-sync. Make sure to update the required fields
5843	* before using them.
5844	*
5845	* @remarks No-long-jump zone!!!
5846	*/
5847	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5848	{
5849	#if HC_ARCH_BITS == 64
5850	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5851	{
5852	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5853	VMMRZCallRing3Disable(pVCpu);
5854	HM_DISABLE_PREEMPT_IF_NEEDED();
5855
5856	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5857	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5858	{
5859	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5860	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5861	}
5862
5863	HM_RESTORE_PREEMPT_IF_NEEDED();
5864	VMMRZCallRing3Enable(pVCpu);
5865	}
5866	else
5867	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5868	#else
5869	NOREF(pMixedCtx);
5870	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5871	#endif
5872
5873	return VINF_SUCCESS;
5874	}
5875
5876
5877	/**
5878	* Saves the auto load/store'd guest MSRs from the current VMCS into
5879	* the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5892	return VINF_SUCCESS;
5893
5894	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5895	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5896	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5897	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5898	{
5899	switch (pMsr->u32Msr)
5900	{
5901	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
5902	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5903	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5904	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5905	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5906	default:
5907	{
5908	AssertFailed();
5909	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5910	}
5911	}
5912	}
5913
5914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5915	return VINF_SUCCESS;
5916	}
5917
5918
5919	/**
5920	* Saves the guest control registers from the current VMCS into the guest-CPU
5921	* context.
5922	*
5923	* @returns VBox status code.
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	*
5929	* @remarks No-long-jump zone!!!
5930	*/
5931	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5932	{
5933	/* Guest CR0. Guest FPU. */
5934	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5935	AssertRCReturn(rc, rc);
5936
5937	/* Guest CR4. */
5938	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5939	AssertRCReturn(rc, rc);
5940
5941	/* Guest CR2 - updated always during the world-switch or in #PF. */
5942	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5943	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5944	{
5945	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5946	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5947
5948	PVM pVM = pVCpu->CTX_SUFF(pVM);
5949	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5950	\|\| ( pVM->hm.s.fNestedPaging
5951	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5952	{
5953	uint64_t u64Val = 0;
5954	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5955	if (pMixedCtx->cr3 != u64Val)
5956	{
5957	CPUMSetGuestCR3(pVCpu, u64Val);
5958	if (VMMRZCallRing3IsEnabled(pVCpu))
5959	{
5960	PGMUpdateCR3(pVCpu, u64Val);
5961	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5962	}
5963	else
5964	{
5965	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5966	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5967	}
5968	}
5969
5970	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5971	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5972	{
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5977
5978	if (VMMRZCallRing3IsEnabled(pVCpu))
5979	{
5980	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5982	}
5983	else
5984	{
5985	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5986	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5987	}
5988	}
5989	}
5990
5991	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
5992	}
5993
5994	/*
5995	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5996	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5997	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5998	*
5999	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6000	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6001	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6002	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6003	*
6004	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6005	*/
6006	if (VMMRZCallRing3IsEnabled(pVCpu))
6007	{
6008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6009	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6010
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6013
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6015	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6016	}
6017
6018	return rc;
6019	}
6020
6021
6022	/**
6023	* Reads a guest segment register from the current VMCS into the guest-CPU
6024	* context.
6025	*
6026	* @returns VBox status code.
6027	* @param pVCpu Pointer to the VMCPU.
6028	* @param idxSel Index of the selector in the VMCS.
6029	* @param idxLimit Index of the segment limit in the VMCS.
6030	* @param idxBase Index of the segment base in the VMCS.
6031	* @param idxAccess Index of the access rights of the segment in the VMCS.
6032	* @param pSelReg Pointer to the segment selector.
6033	*
6034	* @remarks No-long-jump zone!!!
6035	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6036	* macro as that takes care of whether to read from the VMCS cache or
6037	* not.
6038	*/
6039	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6040	PCPUMSELREG pSelReg)
6041	{
6042	NOREF(pVCpu);
6043
6044	uint32_t u32Val = 0;
6045	int rc = VMXReadVmcs32(idxSel, &u32Val);
6046	AssertRCReturn(rc, rc);
6047	pSelReg->Sel = (uint16_t)u32Val;
6048	pSelReg->ValidSel = (uint16_t)u32Val;
6049	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6050
6051	rc = VMXReadVmcs32(idxLimit, &u32Val);
6052	AssertRCReturn(rc, rc);
6053	pSelReg->u32Limit = u32Val;
6054
6055	uint64_t u64Val = 0;
6056	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6057	AssertRCReturn(rc, rc);
6058	pSelReg->u64Base = u64Val;
6059
6060	rc = VMXReadVmcs32(idxAccess, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->Attr.u = u32Val;
6063
6064	/*
6065	* If VT-x marks the segment as unusable, most other bits remain undefined:
6066	* - For CS the L, D and G bits have meaning.
6067	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6068	* - For the remaining data segments no bits are defined.
6069	*
6070	* The present bit and the unusable bit has been observed to be set at the
6071	* same time (the selector was supposed to invalid as we started executing
6072	* a V8086 interrupt in ring-0).
6073	*
6074	* What should be important for the rest of the VBox code, is that the P bit is
6075	* cleared. Some of the other VBox code recognizes the unusable bit, but
6076	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6077	* safe side here, we'll strip off P and other bits we don't care about. If
6078	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6079	*
6080	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6081	*/
6082	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6083	{
6084	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6085
6086	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6087	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6088	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6089
6090	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6091	#ifdef DEBUG_bird
6092	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6093	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6094	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6095	#endif
6096	}
6097	return VINF_SUCCESS;
6098	}
6099
6100
6101	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6102	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6103	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6104	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6105	#else
6106	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6107	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6108	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6109	#endif
6110
6111
6112	/**
6113	* Saves the guest segment registers from the current VMCS into the guest-CPU
6114	* context.
6115	*
6116	* @returns VBox status code.
6117	* @param pVCpu Pointer to the VMCPU.
6118	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6119	* out-of-sync. Make sure to update the required fields
6120	* before using them.
6121	*
6122	* @remarks No-long-jump zone!!!
6123	*/
6124	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6125	{
6126	/* Guest segment registers. */
6127	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6128	{
6129	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6135	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6136
6137	/* Restore segment attributes for real-on-v86 mode hack. */
6138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6139	{
6140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6146	}
6147	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6148	}
6149
6150	return VINF_SUCCESS;
6151	}
6152
6153
6154	/**
6155	* Saves the guest descriptor table registers and task register from the current
6156	* VMCS into the guest-CPU context.
6157	*
6158	* @returns VBox status code.
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6161	* out-of-sync. Make sure to update the required fields
6162	* before using them.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6167	{
6168	int rc = VINF_SUCCESS;
6169
6170	/* Guest LDTR. */
6171	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6172	{
6173	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6174	AssertRCReturn(rc, rc);
6175	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6176	}
6177
6178	/* Guest GDTR. */
6179	uint64_t u64Val = 0;
6180	uint32_t u32Val = 0;
6181	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6182	{
6183	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6185	pMixedCtx->gdtr.pGdt = u64Val;
6186	pMixedCtx->gdtr.cbGdt = u32Val;
6187	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6188	}
6189
6190	/* Guest IDTR. */
6191	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6192	{
6193	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6194	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6195	pMixedCtx->idtr.pIdt = u64Val;
6196	pMixedCtx->idtr.cbIdt = u32Val;
6197	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6198	}
6199
6200	/* Guest TR. */
6201	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6202	{
6203	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6204	AssertRCReturn(rc, rc);
6205
6206	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6207	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208	{
6209	rc = VMXLOCAL_READ_SEG(TR, tr);
6210	AssertRCReturn(rc, rc);
6211	}
6212	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6213	}
6214	return rc;
6215	}
6216
6217	#undef VMXLOCAL_READ_SEG
6218
6219
6220	/**
6221	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6222	* context.
6223	*
6224	* @returns VBox status code.
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	*
6230	* @remarks No-long-jump zone!!!
6231	*/
6232	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6233	{
6234	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6235	{
6236	if (!pVCpu->hm.s.fUsingHyperDR7)
6237	{
6238	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6239	uint32_t u32Val;
6240	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6241	pMixedCtx->dr[7] = u32Val;
6242	}
6243
6244	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6245	}
6246	return VINF_SUCCESS;
6247	}
6248
6249
6250	/**
6251	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6252	*
6253	* @returns VBox status code.
6254	* @param pVCpu Pointer to the VMCPU.
6255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256	* out-of-sync. Make sure to update the required fields
6257	* before using them.
6258	*
6259	* @remarks No-long-jump zone!!!
6260	*/
6261	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262	{
6263	NOREF(pMixedCtx);
6264
6265	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6267	return VINF_SUCCESS;
6268	}
6269
6270
6271	/**
6272	* Saves the entire guest state from the currently active VMCS into the
6273	* guest-CPU context. This essentially VMREADs all guest-data.
6274	*
6275	* @returns VBox status code.
6276	* @param pVCpu Pointer to the VMCPU.
6277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6278	* out-of-sync. Make sure to update the required fields
6279	* before using them.
6280	*/
6281	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	Assert(pVCpu);
6284	Assert(pMixedCtx);
6285
6286	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6287	return VINF_SUCCESS;
6288
6289	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6290	again on the ring-3 callback path, there is no real need to. */
6291	if (VMMRZCallRing3IsEnabled(pVCpu))
6292	VMMR0LogFlushDisable(pVCpu);
6293	else
6294	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6295	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6296
6297	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6328	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6329
6330	if (VMMRZCallRing3IsEnabled(pVCpu))
6331	VMMR0LogFlushEnable(pVCpu);
6332
6333	return rc;
6334	}
6335
6336
6337	/**
6338	* Check per-VM and per-VCPU force flag actions that require us to go back to
6339	* ring-3 for one reason or another.
6340	*
6341	* @returns VBox status code (information status code included).
6342	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6343	* ring-3.
6344	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6345	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6346	* interrupts)
6347	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6348	* all EMTs to be in ring-3.
6349	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6350	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6351	* to the EM loop.
6352	*
6353	* @param pVM Pointer to the VM.
6354	* @param pVCpu Pointer to the VMCPU.
6355	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6356	* out-of-sync. Make sure to update the required fields
6357	* before using them.
6358	*/
6359	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6360	{
6361	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6362
6363	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6364	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6365	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6366	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6367	{
6368	/* We need the control registers now, make sure the guest-CPU context is updated. */
6369	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6370	AssertRCReturn(rc3, rc3);
6371
6372	/* Pending HM CR3 sync. */
6373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6374	{
6375	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6376	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6377	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6378	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6379	}
6380
6381	/* Pending HM PAE PDPEs. */
6382	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6383	{
6384	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6385	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6386	}
6387
6388	/* Pending PGM C3 sync. */
6389	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6390	{
6391	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6392	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6393	if (rc2 != VINF_SUCCESS)
6394	{
6395	AssertRC(rc2);
6396	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6397	return rc2;
6398	}
6399	}
6400
6401	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6402	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6403	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6404	{
6405	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6406	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6407	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6408	return rc2;
6409	}
6410
6411	/* Pending VM request packets, such as hardware interrupts. */
6412	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6413	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6414	{
6415	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6416	return VINF_EM_PENDING_REQUEST;
6417	}
6418
6419	/* Pending PGM pool flushes. */
6420	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6421	{
6422	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6423	return VINF_PGM_POOL_FLUSH_PENDING;
6424	}
6425
6426	/* Pending DMA requests. */
6427	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6428	{
6429	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6430	return VINF_EM_RAW_TO_R3;
6431	}
6432	}
6433
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Converts any TRPM trap into a pending HM event. This is typically used when
6440	* entering from ring-3 (not longjmp returns).
6441	*
6442	* @param pVCpu Pointer to the VMCPU.
6443	*/
6444	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6445	{
6446	Assert(TRPMHasTrap(pVCpu));
6447	Assert(!pVCpu->hm.s.Event.fPending);
6448
6449	uint8_t uVector;
6450	TRPMEVENT enmTrpmEvent;
6451	RTGCUINT uErrCode;
6452	RTGCUINTPTR GCPtrFaultAddress;
6453	uint8_t cbInstr;
6454
6455	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6456	AssertRC(rc);
6457
6458	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6459	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6460	if (enmTrpmEvent == TRPM_TRAP)
6461	{
6462	switch (uVector)
6463	{
6464	case X86_XCPT_BP:
6465	case X86_XCPT_OF:
6466	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6467	break;
6468
6469	case X86_XCPT_PF:
6470	case X86_XCPT_DF:
6471	case X86_XCPT_TS:
6472	case X86_XCPT_NP:
6473	case X86_XCPT_SS:
6474	case X86_XCPT_GP:
6475	case X86_XCPT_AC:
6476	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6477	/* no break! */
6478	default:
6479	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6480	break;
6481	}
6482	}
6483	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6484	{
6485	if (uVector == X86_XCPT_NMI)
6486	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	else
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	}
6490	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6491	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6492	else
6493	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6494
6495	rc = TRPMResetTrap(pVCpu);
6496	AssertRC(rc);
6497	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6498	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6499
6500	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6501	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6502	}
6503
6504
6505	/**
6506	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6507	* VT-x to execute any instruction.
6508	*
6509	* @param pvCpu Pointer to the VMCPU.
6510	*/
6511	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6512	{
6513	Assert(pVCpu->hm.s.Event.fPending);
6514
6515	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6516	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6517	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6518	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6519
6520	/* If a trap was already pending, we did something wrong! */
6521	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6522
6523	TRPMEVENT enmTrapType;
6524	switch (uVectorType)
6525	{
6526	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6527	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6528	enmTrapType = TRPM_HARDWARE_INT;
6529	break;
6530
6531	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6532	enmTrapType = TRPM_SOFTWARE_INT;
6533	break;
6534
6535	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6536	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6538	enmTrapType = TRPM_TRAP;
6539	break;
6540
6541	default:
6542	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6543	enmTrapType = TRPM_32BIT_HACK;
6544	break;
6545	}
6546
6547	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6548
6549	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6550	AssertRC(rc);
6551
6552	if (fErrorCodeValid)
6553	TRPMSetErrorCode(pVCpu, uErrorCode);
6554
6555	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6556	&& uVector == X86_XCPT_PF)
6557	{
6558	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6559	}
6560	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6561	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6562	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6563	{
6564	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6565	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6566	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6567	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6568	}
6569	pVCpu->hm.s.Event.fPending = false;
6570	}
6571
6572
6573	/**
6574	* Does the necessary state syncing before returning to ring-3 for any reason
6575	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6576	*
6577	* @returns VBox status code.
6578	* @param pVM Pointer to the VM.
6579	* @param pVCpu Pointer to the VMCPU.
6580	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6581	* be out-of-sync. Make sure to update the required
6582	* fields before using them.
6583	* @param fSaveGuestState Whether to save the guest state or not.
6584	*
6585	* @remarks No-long-jmp zone!!!
6586	*/
6587	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6588	{
6589	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6590	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6591
6592	RTCPUID idCpu = RTMpCpuId();
6593	Log4Func(("HostCpuId=%u\n", idCpu));
6594
6595	/*
6596	* !!! IMPORTANT !!!
6597	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6598	*/
6599
6600	/* Save the guest state if necessary. */
6601	if ( fSaveGuestState
6602	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6603	{
6604	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6605	AssertRCReturn(rc, rc);
6606	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6607	}
6608
6609	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6610	if (CPUMIsGuestFPUStateActive(pVCpu))
6611	{
6612	/* We shouldn't reload CR0 without saving it first. */
6613	if (!fSaveGuestState)
6614	{
6615	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6616	AssertRCReturn(rc, rc);
6617	}
6618	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6619	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6620	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6621	}
6622
6623	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6624	#ifdef VBOX_STRICT
6625	if (CPUMIsHyperDebugStateActive(pVCpu))
6626	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6627	#endif
6628	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6629	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6630	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6631	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6632
6633	#if HC_ARCH_BITS == 64
6634	/* Restore host-state bits that VT-x only restores partially. */
6635	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6636	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6637	{
6638	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6639	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6640	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6641	}
6642	#endif
6643
6644	#if HC_ARCH_BITS == 64
6645	/* Restore the host MSRs as we're leaving VT-x context. */
6646	if ( pVM->hm.s.fAllow64BitGuests
6647	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6648	{
6649	/* We shouldn't reload the guest MSRs without saving it first. */
6650	if (!fSaveGuestState)
6651	{
6652	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6653	AssertRCReturn(rc, rc);
6654	}
6655	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6656	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6657	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6658	}
6659	#endif
6660
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6665	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6666	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6667	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6669
6670	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6671
6672	/** @todo This partially defeats the purpose of having preemption hooks.
6673	* The problem is, deregistering the hooks should be moved to a place that
6674	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6675	* context.
6676	*/
6677	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6678	{
6679	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6680	AssertRCReturn(rc, rc);
6681
6682	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6683	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6684	}
6685	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6686	NOREF(idCpu);
6687
6688	return VINF_SUCCESS;
6689	}
6690
6691
6692	/**
6693	* Leaves the VT-x session.
6694	*
6695	* @returns VBox status code.
6696	* @param pVM Pointer to the VM.
6697	* @param pVCpu Pointer to the VMCPU.
6698	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6699	* out-of-sync. Make sure to update the required fields
6700	* before using them.
6701	*
6702	* @remarks No-long-jmp zone!!!
6703	*/
6704	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6705	{
6706	HM_DISABLE_PREEMPT_IF_NEEDED();
6707	HMVMX_ASSERT_CPU_SAFE();
6708	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6710
6711	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6712	and done this from the VMXR0ThreadCtxCallback(). */
6713	if (!pVCpu->hm.s.fLeaveDone)
6714	{
6715	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6716	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6717	pVCpu->hm.s.fLeaveDone = true;
6718	}
6719
6720	/*
6721	* !!! IMPORTANT !!!
6722	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6723	*/
6724
6725	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6726	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6727	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6728	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6729	VMMR0ThreadCtxHooksDeregister(pVCpu);
6730
6731	/* Leave HM context. This takes care of local init (term). */
6732	int rc = HMR0LeaveCpu(pVCpu);
6733
6734	HM_RESTORE_PREEMPT_IF_NEEDED();
6735
6736	return rc;
6737	}
6738
6739
6740	/**
6741	* Does the necessary state syncing before doing a longjmp to ring-3.
6742	*
6743	* @returns VBox status code.
6744	* @param pVM Pointer to the VM.
6745	* @param pVCpu Pointer to the VMCPU.
6746	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6747	* out-of-sync. Make sure to update the required fields
6748	* before using them.
6749	*
6750	* @remarks No-long-jmp zone!!!
6751	*/
6752	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6753	{
6754	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6755	}
6756
6757
6758	/**
6759	* Take necessary actions before going back to ring-3.
6760	*
6761	* An action requires us to go back to ring-3. This function does the necessary
6762	* steps before we can safely return to ring-3. This is not the same as longjmps
6763	* to ring-3, this is voluntary and prepares the guest so it may continue
6764	* executing outside HM (recompiler/IEM).
6765	*
6766	* @returns VBox status code.
6767	* @param pVM Pointer to the VM.
6768	* @param pVCpu Pointer to the VMCPU.
6769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6770	* out-of-sync. Make sure to update the required fields
6771	* before using them.
6772	* @param rcExit The reason for exiting to ring-3. Can be
6773	* VINF_VMM_UNKNOWN_RING3_CALL.
6774	*/
6775	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6776	{
6777	Assert(pVM);
6778	Assert(pVCpu);
6779	Assert(pMixedCtx);
6780	HMVMX_ASSERT_PREEMPT_SAFE();
6781
6782	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6783	{
6784	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6785	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6786	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6787	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6788	}
6789
6790	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6791	VMMRZCallRing3Disable(pVCpu);
6792	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6793
6794	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6795	if (pVCpu->hm.s.Event.fPending)
6796	{
6797	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6798	Assert(!pVCpu->hm.s.Event.fPending);
6799	}
6800
6801	/* Save guest state and restore host state bits. */
6802	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6803	AssertRCReturn(rc, rc);
6804	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6805
6806	/* Sync recompiler state. */
6807	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6808	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6809	\| CPUM_CHANGED_LDTR
6810	\| CPUM_CHANGED_GDTR
6811	\| CPUM_CHANGED_IDTR
6812	\| CPUM_CHANGED_TR
6813	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6814	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6815	if ( pVM->hm.s.fNestedPaging
6816	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6817	{
6818	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6819	}
6820
6821	Assert(!pVCpu->hm.s.fClearTrapFlag);
6822
6823	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6824	if (rcExit != VINF_EM_RAW_INTERRUPT)
6825	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6826
6827	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6828
6829	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6830	VMMRZCallRing3RemoveNotification(pVCpu);
6831	VMMRZCallRing3Enable(pVCpu);
6832
6833	return rc;
6834	}
6835
6836
6837	/**
6838	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6839	* longjump to ring-3 and possibly get preempted.
6840	*
6841	* @returns VBox status code.
6842	* @param pVCpu Pointer to the VMCPU.
6843	* @param enmOperation The operation causing the ring-3 longjump.
6844	* @param pvUser Opaque pointer to the guest-CPU context. The data
6845	* may be out-of-sync. Make sure to update the required
6846	* fields before using them.
6847	*/
6848	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6849	{
6850	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6851	{
6852	/*
6853	* !!! IMPORTANT !!!
6854	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6855	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6856	*/
6857	VMMRZCallRing3RemoveNotification(pVCpu);
6858	VMMRZCallRing3Disable(pVCpu);
6859	HM_DISABLE_PREEMPT_IF_NEEDED();
6860
6861	PVM pVM = pVCpu->CTX_SUFF(pVM);
6862	if (CPUMIsGuestFPUStateActive(pVCpu))
6863	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6864
6865	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6866
6867	#if HC_ARCH_BITS == 64
6868	/* Restore host-state bits that VT-x only restores partially. */
6869	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6870	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6871	{
6872	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6873	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6874	}
6875	/* Restore the host MSRs as we're leaving VT-x context. */
6876	if ( pVM->hm.s.fAllow64BitGuests
6877	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6878	{
6879	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6880	}
6881	#endif
6882	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6883	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6884	{
6885	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6886	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6887	}
6888
6889	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6890	VMMR0ThreadCtxHooksDeregister(pVCpu);
6891
6892	HMR0LeaveCpu(pVCpu);
6893	HM_RESTORE_PREEMPT_IF_NEEDED();
6894	return VINF_SUCCESS;
6895	}
6896
6897	Assert(pVCpu);
6898	Assert(pvUser);
6899	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6900	HMVMX_ASSERT_PREEMPT_SAFE();
6901
6902	VMMRZCallRing3Disable(pVCpu);
6903	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6904
6905	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6906	enmOperation));
6907
6908	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6909	AssertRCReturn(rc, rc);
6910
6911	VMMRZCallRing3Enable(pVCpu);
6912	return VINF_SUCCESS;
6913	}
6914
6915
6916	/**
6917	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6918	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6919	*
6920	* @param pVCpu Pointer to the VMCPU.
6921	*/
6922	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6923	{
6924	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6925	{
6926	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6927	{
6928	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6929	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6930	AssertRC(rc);
6931	}
6932	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6933	}
6934
6935
6936	/**
6937	* Evaluates the event to be delivered to the guest and sets it as the pending
6938	* event.
6939	*
6940	* @param pVCpu Pointer to the VMCPU.
6941	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6942	* out-of-sync. Make sure to update the required fields
6943	* before using them.
6944	*/
6945	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6946	{
6947	Assert(!pVCpu->hm.s.Event.fPending);
6948
6949	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6950	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6951	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6952	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6953
6954	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6955	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6956	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6957	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6958	Assert(!TRPMHasTrap(pVCpu));
6959
6960	/** @todo SMI. SMIs take priority over NMIs. */
6961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6962	{
6963	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6964	if ( !fBlockMovSS
6965	&& !fBlockSti)
6966	{
6967	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6968	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6969	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6970	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6971
6972	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6973	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6974	}
6975	else
6976	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6977	}
6978	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6979	&& !pVCpu->hm.s.fSingleInstruction)
6980	{
6981	/*
6982	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6983	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6984	* evaluated here and not set as pending, solely based on the force-flags.
6985	*/
6986	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6987	AssertRC(rc);
6988	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6989	if ( !fBlockInt
6990	&& !fBlockSti
6991	&& !fBlockMovSS)
6992	{
6993	uint8_t u8Interrupt;
6994	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6995	if (RT_SUCCESS(rc))
6996	{
6997	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6998	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6999	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7000
7001	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7002	}
7003	else
7004	{
7005	/** @todo Does this actually happen? If not turn it into an assertion. */
7006	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7007	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7008	}
7009	}
7010	else
7011	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7012	}
7013	}
7014
7015
7016	/**
7017	* Injects any pending events into the guest if the guest is in a state to
7018	* receive them.
7019	*
7020	* @returns VBox status code (informational status codes included).
7021	* @param pVCpu Pointer to the VMCPU.
7022	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7023	* out-of-sync. Make sure to update the required fields
7024	* before using them.
7025	*/
7026	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7027	{
7028	HMVMX_ASSERT_PREEMPT_SAFE();
7029	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7030
7031	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7032	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7033	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7034	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7035
7036	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7037	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7038	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7039	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7040	Assert(!TRPMHasTrap(pVCpu));
7041
7042	int rc = VINF_SUCCESS;
7043	if (pVCpu->hm.s.Event.fPending)
7044	{
7045	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7046	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7047	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7048	{
7049	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7050	AssertRCReturn(rc, rc);
7051	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7052	Assert(!fBlockInt);
7053	Assert(!fBlockSti);
7054	Assert(!fBlockMovSS);
7055	}
7056	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7057	{
7058	Assert(!fBlockSti);
7059	Assert(!fBlockMovSS);
7060	}
7061	#endif
7062	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7063	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7064	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7065	AssertRCReturn(rc, rc);
7066
7067	/* Update the interruptibility-state as it could have been changed by
7068	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7069	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7070	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7071
7072	#ifdef VBOX_WITH_STATISTICS
7073	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7074	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7075	else
7076	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7077	#endif
7078	}
7079
7080	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7081	if ( fBlockSti
7082	\|\| fBlockMovSS)
7083	{
7084	if ( !pVCpu->hm.s.fSingleInstruction
7085	&& !DBGFIsStepping(pVCpu))
7086	{
7087	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7088	AssertRCReturn(rc2, rc2);
7089	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7090	{
7091	/*
7092	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7093	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7094	* See Intel spec. 27.3.4 "Saving Non-Register State".
7095	*/
7096	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7097	AssertRCReturn(rc2, rc2);
7098	}
7099	}
7100	else if (pMixedCtx->eflags.Bits.u1TF)
7101	{
7102	/*
7103	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7104	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7105	*/
7106	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7107	uIntrState = 0;
7108	}
7109	}
7110
7111	/*
7112	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7113	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7114	*/
7115	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7116	AssertRC(rc2);
7117
7118	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7119	NOREF(fBlockMovSS); NOREF(fBlockSti);
7120	return rc;
7121	}
7122
7123
7124	/**
7125	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7126	*
7127	* @param pVCpu Pointer to the VMCPU.
7128	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7129	* out-of-sync. Make sure to update the required fields
7130	* before using them.
7131	*/
7132	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7133	{
7134	NOREF(pMixedCtx);
7135	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7136	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7137	}
7138
7139
7140	/**
7141	* Injects a double-fault (#DF) exception into the VM.
7142	*
7143	* @returns VBox status code (informational status code included).
7144	* @param pVCpu Pointer to the VMCPU.
7145	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7146	* out-of-sync. Make sure to update the required fields
7147	* before using them.
7148	*/
7149	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7150	{
7151	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7152	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7153	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7154	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7155	puIntrState);
7156	}
7157
7158
7159	/**
7160	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7161	*
7162	* @param pVCpu Pointer to the VMCPU.
7163	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7164	* out-of-sync. Make sure to update the required fields
7165	* before using them.
7166	*/
7167	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7168	{
7169	NOREF(pMixedCtx);
7170	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7171	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7172	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7173	}
7174
7175
7176	/**
7177	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7178	*
7179	* @param pVCpu Pointer to the VMCPU.
7180	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7181	* out-of-sync. Make sure to update the required fields
7182	* before using them.
7183	* @param cbInstr The value of RIP that is to be pushed on the guest
7184	* stack.
7185	*/
7186	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7187	{
7188	NOREF(pMixedCtx);
7189	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7190	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7191	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7192	}
7193
7194
7195	/**
7196	* Injects a general-protection (#GP) fault into the VM.
7197	*
7198	* @returns VBox status code (informational status code included).
7199	* @param pVCpu Pointer to the VMCPU.
7200	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7201	* out-of-sync. Make sure to update the required fields
7202	* before using them.
7203	* @param u32ErrorCode The error code associated with the #GP.
7204	*/
7205	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7206	uint32_t *puIntrState)
7207	{
7208	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7209	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7210	if (fErrorCodeValid)
7211	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7212	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7213	puIntrState);
7214	}
7215
7216
7217	/**
7218	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7219	*
7220	* @param pVCpu Pointer to the VMCPU.
7221	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7222	* out-of-sync. Make sure to update the required fields
7223	* before using them.
7224	* @param uVector The software interrupt vector number.
7225	* @param cbInstr The value of RIP that is to be pushed on the guest
7226	* stack.
7227	*/
7228	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7229	{
7230	NOREF(pMixedCtx);
7231	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7232	if ( uVector == X86_XCPT_BP
7233	\|\| uVector == X86_XCPT_OF)
7234	{
7235	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7236	}
7237	else
7238	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7239	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7240	}
7241
7242
7243	/**
7244	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7245	* stack.
7246	*
7247	* @returns VBox status code (information status code included).
7248	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7249	* @param pVM Pointer to the VM.
7250	* @param pMixedCtx Pointer to the guest-CPU context.
7251	* @param uValue The value to push to the guest stack.
7252	*/
7253	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7254	{
7255	/*
7256	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7257	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7258	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7259	*/
7260	if (pMixedCtx->sp == 1)
7261	return VINF_EM_RESET;
7262	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7263	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7264	AssertRCReturn(rc, rc);
7265	return rc;
7266	}
7267
7268
7269	/**
7270	* Injects an event into the guest upon VM-entry by updating the relevant fields
7271	* in the VM-entry area in the VMCS.
7272	*
7273	* @returns VBox status code (informational error codes included).
7274	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7275	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7276	*
7277	* @param pVCpu Pointer to the VMCPU.
7278	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7279	* be out-of-sync. Make sure to update the required
7280	* fields before using them.
7281	* @param u64IntInfo The VM-entry interruption-information field.
7282	* @param cbInstr The VM-entry instruction length in bytes (for
7283	* software interrupts, exceptions and privileged
7284	* software exceptions).
7285	* @param u32ErrCode The VM-entry exception error code.
7286	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7287	* @param puIntrState Pointer to the current guest interruptibility-state.
7288	* This interruptibility-state will be updated if
7289	* necessary. This cannot not be NULL.
7290	*
7291	* @remarks Requires CR0!
7292	* @remarks No-long-jump zone!!!
7293	*/
7294	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7295	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7296	{
7297	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7298	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7299	Assert(puIntrState);
7300	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7301
7302	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7303	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7304
7305	#ifdef VBOX_STRICT
7306	/* Validate the error-code-valid bit for hardware exceptions. */
7307	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7308	{
7309	switch (uVector)
7310	{
7311	case X86_XCPT_PF:
7312	case X86_XCPT_DF:
7313	case X86_XCPT_TS:
7314	case X86_XCPT_NP:
7315	case X86_XCPT_SS:
7316	case X86_XCPT_GP:
7317	case X86_XCPT_AC:
7318	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7319	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7320	/* fallthru */
7321	default:
7322	break;
7323	}
7324	}
7325	#endif
7326
7327	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7328	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7329	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7330
7331	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7332
7333	/* We require CR0 to check if the guest is in real-mode. */
7334	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7335	AssertRCReturn(rc, rc);
7336
7337	/*
7338	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7339	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7340	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7341	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7342	*/
7343	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7344	{
7345	PVM pVM = pVCpu->CTX_SUFF(pVM);
7346	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7347	{
7348	Assert(PDMVmmDevHeapIsEnabled(pVM));
7349	Assert(pVM->hm.s.vmx.pRealModeTSS);
7350
7351	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7352	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7353	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7354	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7355	AssertRCReturn(rc, rc);
7356	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7357
7358	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7359	const size_t cbIdtEntry = sizeof(X86IDTR16);
7360	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7361	{
7362	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7363	if (uVector == X86_XCPT_DF)
7364	return VINF_EM_RESET;
7365	else if (uVector == X86_XCPT_GP)
7366	{
7367	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7368	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7369	}
7370
7371	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7372	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7373	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7374	}
7375
7376	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7377	uint16_t uGuestIp = pMixedCtx->ip;
7378	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7379	{
7380	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7381	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7382	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7383	}
7384	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7385	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7386
7387	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7388	X86IDTR16 IdtEntry;
7389	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7390	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7391	AssertRCReturn(rc, rc);
7392
7393	/* Construct the stack frame for the interrupt/exception handler. */
7394	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7395	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7396	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7397	AssertRCReturn(rc, rc);
7398
7399	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7400	if (rc == VINF_SUCCESS)
7401	{
7402	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7403	pMixedCtx->rip = IdtEntry.offSel;
7404	pMixedCtx->cs.Sel = IdtEntry.uSel;
7405	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7406	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7407	&& uVector == X86_XCPT_PF)
7408	{
7409	pMixedCtx->cr2 = GCPtrFaultAddress;
7410	}
7411
7412	/* If any other guest-state bits are changed here, make sure to update
7413	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7414	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7415	\| HM_CHANGED_GUEST_RIP
7416	\| HM_CHANGED_GUEST_RFLAGS
7417	\| HM_CHANGED_GUEST_RSP);
7418
7419	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7420	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7421	{
7422	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7423	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7424	Log4(("Clearing inhibition due to STI.\n"));
7425	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7426	}
7427	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7428
7429	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7430	it, if we are returning to ring-3 before executing guest code. */
7431	pVCpu->hm.s.Event.fPending = false;
7432	}
7433	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7434	return rc;
7435	}
7436	else
7437	{
7438	/*
7439	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7440	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7441	*/
7442	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7443	}
7444	}
7445
7446	/* Validate. */
7447	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7448	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7449	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7450
7451	/* Inject. */
7452	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7453	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7454	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7455	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7456
7457	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7458	&& uVector == X86_XCPT_PF)
7459	{
7460	pMixedCtx->cr2 = GCPtrFaultAddress;
7461	}
7462
7463	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7464	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7465
7466	AssertRCReturn(rc, rc);
7467	return rc;
7468	}
7469
7470
7471	/**
7472	* Clears the interrupt-window exiting control in the VMCS and if necessary
7473	* clears the current event in the VMCS as well.
7474	*
7475	* @returns VBox status code.
7476	* @param pVCpu Pointer to the VMCPU.
7477	*
7478	* @remarks Use this function only to clear events that have not yet been
7479	* delivered to the guest but are injected in the VMCS!
7480	* @remarks No-long-jump zone!!!
7481	*/
7482	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7483	{
7484	int rc;
7485	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7486
7487	/* Clear interrupt-window exiting control. */
7488	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7489	{
7490	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7491	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7492	AssertRC(rc);
7493	}
7494
7495	if (!pVCpu->hm.s.Event.fPending)
7496	return;
7497
7498	#ifdef VBOX_STRICT
7499	uint32_t u32EntryInfo;
7500	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7501	AssertRC(rc);
7502	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7503	#endif
7504
7505	/* Clear the entry-interruption field (including the valid bit). */
7506	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7507	AssertRC(rc);
7508
7509	/* Clear the pending debug exception field. */
7510	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7511	AssertRC(rc);
7512	}
7513
7514
7515	/**
7516	* Enters the VT-x session.
7517	*
7518	* @returns VBox status code.
7519	* @param pVM Pointer to the VM.
7520	* @param pVCpu Pointer to the VMCPU.
7521	* @param pCpu Pointer to the CPU info struct.
7522	*/
7523	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7524	{
7525	AssertPtr(pVM);
7526	AssertPtr(pVCpu);
7527	Assert(pVM->hm.s.vmx.fSupported);
7528	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7529	NOREF(pCpu); NOREF(pVM);
7530
7531	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7532	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7533
7534	#ifdef VBOX_STRICT
7535	/* Make sure we're in VMX root mode. */
7536	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7537	if (!(u32HostCR4 & X86_CR4_VMXE))
7538	{
7539	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7540	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7541	}
7542	#endif
7543
7544	/*
7545	* Load the VCPU's VMCS as the current (and active) one.
7546	*/
7547	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7548	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7549	if (RT_FAILURE(rc))
7550	return rc;
7551
7552	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7553	pVCpu->hm.s.fLeaveDone = false;
7554	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7555
7556	return VINF_SUCCESS;
7557	}
7558
7559
7560	/**
7561	* The thread-context callback (only on platforms which support it).
7562	*
7563	* @param enmEvent The thread-context event.
7564	* @param pVCpu Pointer to the VMCPU.
7565	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7566	* @thread EMT(pVCpu)
7567	*/
7568	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7569	{
7570	NOREF(fGlobalInit);
7571
7572	switch (enmEvent)
7573	{
7574	case RTTHREADCTXEVENT_PREEMPTING:
7575	{
7576	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7577	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7578	VMCPU_ASSERT_EMT(pVCpu);
7579
7580	PVM pVM = pVCpu->CTX_SUFF(pVM);
7581	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7582
7583	/* No longjmps (logger flushes, locks) in this fragile context. */
7584	VMMRZCallRing3Disable(pVCpu);
7585	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7586
7587	/*
7588	* Restore host-state (FPU, debug etc.)
7589	*/
7590	if (!pVCpu->hm.s.fLeaveDone)
7591	{
7592	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7593	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7594	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7595	pVCpu->hm.s.fLeaveDone = true;
7596	}
7597
7598	/* Leave HM context, takes care of local init (term). */
7599	int rc = HMR0LeaveCpu(pVCpu);
7600	AssertRC(rc); NOREF(rc);
7601
7602	/* Restore longjmp state. */
7603	VMMRZCallRing3Enable(pVCpu);
7604	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7605	break;
7606	}
7607
7608	case RTTHREADCTXEVENT_RESUMED:
7609	{
7610	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7611	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7612	VMCPU_ASSERT_EMT(pVCpu);
7613
7614	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7615	VMMRZCallRing3Disable(pVCpu);
7616	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7617
7618	/* Initialize the bare minimum state required for HM. This takes care of
7619	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7620	int rc = HMR0EnterCpu(pVCpu);
7621	AssertRC(rc);
7622	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7623
7624	/* Load the active VMCS as the current one. */
7625	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7626	{
7627	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7628	AssertRC(rc); NOREF(rc);
7629	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7630	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7631	}
7632	pVCpu->hm.s.fLeaveDone = false;
7633
7634	/* Restore longjmp state. */
7635	VMMRZCallRing3Enable(pVCpu);
7636	break;
7637	}
7638
7639	default:
7640	break;
7641	}
7642	}
7643
7644
7645	/**
7646	* Saves the host state in the VMCS host-state.
7647	* Sets up the VM-exit MSR-load area.
7648	*
7649	* The CPU state will be loaded from these fields on every successful VM-exit.
7650	*
7651	* @returns VBox status code.
7652	* @param pVM Pointer to the VM.
7653	* @param pVCpu Pointer to the VMCPU.
7654	*
7655	* @remarks No-long-jump zone!!!
7656	*/
7657	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7658	{
7659	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7660
7661	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7662	return VINF_SUCCESS;
7663
7664	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7665	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7666
7667	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7668	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7669
7670	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7671	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7672
7673	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7674	return rc;
7675	}
7676
7677
7678	/**
7679	* Saves the host state in the VMCS host-state.
7680	*
7681	* @returns VBox status code.
7682	* @param pVM Pointer to the VM.
7683	* @param pVCpu Pointer to the VMCPU.
7684	*
7685	* @remarks No-long-jump zone!!!
7686	*/
7687	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7688	{
7689	AssertPtr(pVM);
7690	AssertPtr(pVCpu);
7691
7692	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7693
7694	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7695	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7696	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7697	return hmR0VmxSaveHostState(pVM, pVCpu);
7698	}
7699
7700
7701	/**
7702	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7703	* loaded from these fields on every successful VM-entry.
7704	*
7705	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7706	* Sets up the VM-entry controls.
7707	* Sets up the appropriate VMX non-root function to execute guest code based on
7708	* the guest CPU mode.
7709	*
7710	* @returns VBox status code.
7711	* @param pVM Pointer to the VM.
7712	* @param pVCpu Pointer to the VMCPU.
7713	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7714	* out-of-sync. Make sure to update the required fields
7715	* before using them.
7716	*
7717	* @remarks No-long-jump zone!!!
7718	*/
7719	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7720	{
7721	AssertPtr(pVM);
7722	AssertPtr(pVCpu);
7723	AssertPtr(pMixedCtx);
7724	HMVMX_ASSERT_PREEMPT_SAFE();
7725
7726	#ifdef LOG_ENABLED
7727	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7728	* probably not initialized yet? Anyway this will do for now.
7729	*
7730	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7731	* interface and disable ring-3 calls when thread-context hooks are not
7732	* available. */
7733	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7734	VMMR0LogFlushDisable(pVCpu);
7735	#endif
7736
7737	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7738
7739	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7740
7741	/* Determine real-on-v86 mode. */
7742	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7743	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7744	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7745	{
7746	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7747	}
7748
7749	/*
7750	* Load the guest-state into the VMCS.
7751	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7752	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7753	*/
7754	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7755	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7756
7757	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7758	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7760
7761	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7762	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7763	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7764
7765	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7766	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7767
7768	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7769	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7770
7771	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7772	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7773	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7774
7775	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7776	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7777
7778	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7779	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7780
7781	/*
7782	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7783	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7784	*/
7785	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7786	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7787
7788	/* Clear any unused and reserved bits. */
7789	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7790
7791	#ifdef LOG_ENABLED
7792	/* Only reenable log-flushing if the caller has it enabled. */
7793	if (!fCallerDisabledLogFlush)
7794	VMMR0LogFlushEnable(pVCpu);
7795	#endif
7796
7797	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7798	return rc;
7799	}
7800
7801
7802	/**
7803	* Loads the state shared between the host and guest into the VMCS.
7804	*
7805	* @param pVM Pointer to the VM.
7806	* @param pVCpu Pointer to the VMCPU.
7807	* @param pCtx Pointer to the guest-CPU context.
7808	*
7809	* @remarks No-long-jump zone!!!
7810	*/
7811	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7812	{
7813	NOREF(pVM);
7814
7815	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7816	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7817
7818	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7819	{
7820	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7821	AssertRC(rc);
7822	}
7823
7824	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7825	{
7826	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7827	AssertRC(rc);
7828
7829	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7830	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7831	{
7832	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7833	AssertRC(rc);
7834	}
7835	}
7836
7837	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7838	{
7839	#if HC_ARCH_BITS == 64
7840	if (pVM->hm.s.fAllow64BitGuests)
7841	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7842	#endif
7843	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7844	}
7845
7846	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7847	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7848	}
7849
7850
7851	/**
7852	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7853	*
7854	* @param pVM Pointer to the VM.
7855	* @param pVCpu Pointer to the VMCPU.
7856	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7857	* out-of-sync. Make sure to update the required fields
7858	* before using them.
7859	*/
7860	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7861	{
7862	HMVMX_ASSERT_PREEMPT_SAFE();
7863
7864	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7865	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7866	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7867	#endif
7868
7869	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7870	{
7871	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7872	AssertRC(rc);
7873	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7874	}
7875	else if (HMCPU_CF_VALUE(pVCpu))
7876	{
7877	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7878	AssertRC(rc);
7879	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7880	}
7881
7882	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7883	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7884	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7885	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7886
7887	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7888	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7889	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7890	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7891	#endif
7892	}
7893
7894
7895	/**
7896	* Does the preparations before executing guest code in VT-x.
7897	*
7898	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7899	* recompiler. We must be cautious what we do here regarding committing
7900	* guest-state information into the VMCS assuming we assuredly execute the
7901	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7902	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7903	* so that the recompiler can (and should) use them when it resumes guest
7904	* execution. Otherwise such operations must be done when we can no longer
7905	* exit to ring-3.
7906	*
7907	* @returns Strict VBox status code.
7908	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7909	* have been disabled.
7910	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7911	* double-fault into the guest.
7912	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7913	*
7914	* @param pVM Pointer to the VM.
7915	* @param pVCpu Pointer to the VMCPU.
7916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7917	* out-of-sync. Make sure to update the required fields
7918	* before using them.
7919	* @param pVmxTransient Pointer to the VMX transient structure.
7920	*/
7921	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7922	{
7923	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7924
7925	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7926	PGMRZDynMapFlushAutoSet(pVCpu);
7927	#endif
7928
7929	/* Check force flag actions that might require us to go back to ring-3. */
7930	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7931	if (rc != VINF_SUCCESS)
7932	return rc;
7933
7934	#ifndef IEM_VERIFICATION_MODE_FULL
7935	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7936	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7937	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7938	{
7939	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7940	RTGCPHYS GCPhysApicBase;
7941	GCPhysApicBase = pMixedCtx->msrApicBase;
7942	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7943
7944	/* Unalias any existing mapping. */
7945	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7946	AssertRCReturn(rc, rc);
7947
7948	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7949	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7950	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7951	AssertRCReturn(rc, rc);
7952
7953	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7954	}
7955	#endif /* !IEM_VERIFICATION_MODE_FULL */
7956
7957	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7958	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7959
7960	/*
7961	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7962	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7963	*/
7964	if (TRPMHasTrap(pVCpu))
7965	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7966	else if (!pVCpu->hm.s.Event.fPending)
7967	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7968
7969	/*
7970	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7971	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7972	*/
7973	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7974	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7975	{
7976	Assert(rc == VINF_EM_RESET);
7977	return rc;
7978	}
7979
7980	/*
7981	* No longjmps to ring-3 from this point on!!!
7982	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7983	* This also disables flushing of the R0-logger instance (if any).
7984	*/
7985	VMMRZCallRing3Disable(pVCpu);
7986
7987	/*
7988	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7989	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7990	*
7991	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7992	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7993	*
7994	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7995	* executing guest code.
7996	*/
7997	pVmxTransient->uEflags = ASMIntDisableFlags();
7998	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7999	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8000	{
8001	hmR0VmxClearEventVmcs(pVCpu);
8002	ASMSetFlags(pVmxTransient->uEflags);
8003	VMMRZCallRing3Enable(pVCpu);
8004	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8005	return VINF_EM_RAW_TO_R3;
8006	}
8007	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8008	{
8009	hmR0VmxClearEventVmcs(pVCpu);
8010	ASMSetFlags(pVmxTransient->uEflags);
8011	VMMRZCallRing3Enable(pVCpu);
8012	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8013	return VINF_EM_RAW_INTERRUPT;
8014	}
8015
8016	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8017	pVCpu->hm.s.Event.fPending = false;
8018
8019	return VINF_SUCCESS;
8020	}
8021
8022
8023	/**
8024	* Prepares to run guest code in VT-x and we've committed to doing so. This
8025	* means there is no backing out to ring-3 or anywhere else at this
8026	* point.
8027	*
8028	* @param pVM Pointer to the VM.
8029	* @param pVCpu Pointer to the VMCPU.
8030	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8031	* out-of-sync. Make sure to update the required fields
8032	* before using them.
8033	* @param pVmxTransient Pointer to the VMX transient structure.
8034	*
8035	* @remarks Called with preemption disabled.
8036	* @remarks No-long-jump zone!!!
8037	*/
8038	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8039	{
8040	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8041	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8042	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8043
8044	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8045	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8046
8047	/*
8048	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8049	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8050	* Reload only the necessary state, the assertion will catch if other parts of the code
8051	* change.
8052	*/
8053	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8054	{
8055	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8056	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8057	}
8058
8059	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8060	if (!CPUMIsGuestFPUStateActive(pVCpu))
8061	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8062	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8063	#endif
8064
8065	if ( pVCpu->hm.s.fUseGuestFpu
8066	&& !CPUMIsGuestFPUStateActive(pVCpu))
8067	{
8068	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8069	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8070	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8071	}
8072
8073	/*
8074	* The host MSR values the very first time around won't be updated, so we need to
8075	* fill those values in. Subsequently, it's updated as part of the host state.
8076	*/
8077	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8078	&& pVCpu->hm.s.vmx.cMsrs > 0)
8079	{
8080	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8081	}
8082
8083	/*
8084	* Load the host state bits as we may've been preempted (only happens when
8085	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8086	*/
8087	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8088	{
8089	/* This ASSUMES that pfnStartVM has been set up already. */
8090	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8091	AssertRC(rc);
8092	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8093	}
8094	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8095
8096	/*
8097	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8098	*/
8099	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8100	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8101	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8102
8103	/* Store status of the shared guest-host state at the time of VM-entry. */
8104	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8105	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8106	{
8107	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8108	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8109	}
8110	else
8111	#endif
8112	{
8113	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8114	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8115	}
8116	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8117
8118	/*
8119	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8120	*/
8121	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8122	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8123
8124	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8125	RTCPUID idCurrentCpu = pCpu->idCpu;
8126	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8127	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8128	{
8129	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8130	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8131	}
8132
8133	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8134	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8135	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8136	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8137
8138	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8139
8140	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8141	to start executing. */
8142
8143	/*
8144	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8145	*/
8146	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8147	{
8148	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8149	{
8150	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8151	AssertRC(rc2);
8152	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8153	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */);
8154	}
8155	else
8156	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8157	}
8158	#ifdef VBOX_STRICT
8159	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8160	#endif
8161	}
8162
8163
8164	/**
8165	* Performs some essential restoration of state after running guest code in
8166	* VT-x.
8167	*
8168	* @param pVM Pointer to the VM.
8169	* @param pVCpu Pointer to the VMCPU.
8170	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8171	* out-of-sync. Make sure to update the required fields
8172	* before using them.
8173	* @param pVmxTransient Pointer to the VMX transient structure.
8174	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8175	*
8176	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8177	*
8178	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8179	* unconditionally when it is safe to do so.
8180	*/
8181	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8182	{
8183	NOREF(pVM);
8184
8185	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8186
8187	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8188	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8189	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8190	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8191	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8192
8193	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8194	{
8195	/** @todo Find a way to fix hardcoding a guestimate. */
8196	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8197	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8198	}
8199
8200	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8201	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8202	Assert(!(ASMGetFlags() & X86_EFL_IF));
8203	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8204
8205	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8206	if (CPUMIsGuestFPUStateActive(pVCpu))
8207	{
8208	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8209	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8210	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8211	}
8212	#endif
8213
8214	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8215	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8216	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8217	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8218
8219	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8220	uint32_t uExitReason;
8221	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8222	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8223	AssertRC(rc);
8224	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8225	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8226
8227	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8228	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8229	{
8230	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8231	pVmxTransient->fVMEntryFailed));
8232	return;
8233	}
8234
8235	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8236	{
8237	/* Update the guest interruptibility-state from the VMCS. */
8238	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8239	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8240	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8241	AssertRC(rc);
8242	#endif
8243	/*
8244	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8245	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8246	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8247	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8248	*/
8249	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8250	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8251	{
8252	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8253	AssertRC(rc);
8254	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8255	}
8256	}
8257	}
8258
8259
8260
8261	/**
8262	* Runs the guest code using VT-x the normal way.
8263	*
8264	* @returns VBox status code.
8265	* @param pVM Pointer to the VM.
8266	* @param pVCpu Pointer to the VMCPU.
8267	* @param pCtx Pointer to the guest-CPU context.
8268	*
8269	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8270	*/
8271	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8272	{
8273	VMXTRANSIENT VmxTransient;
8274	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8275	int rc = VERR_INTERNAL_ERROR_5;
8276	uint32_t cLoops = 0;
8277
8278	for (;; cLoops++)
8279	{
8280	Assert(!HMR0SuspendPending());
8281	HMVMX_ASSERT_CPU_SAFE();
8282
8283	/* Preparatory work for running guest code, this may force us to return
8284	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8285	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8286	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8287	if (rc != VINF_SUCCESS)
8288	break;
8289
8290	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8291	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8292	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8293
8294	/* Restore any residual host-state and save any bits shared between host
8295	and guest into the guest-CPU state. Re-enables interrupts! */
8296	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8297
8298	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8299	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8300	{
8301	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8302	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8303	return rc;
8304	}
8305
8306	/* Handle the VM-exit. */
8307	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8309	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8310	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8311	HMVMX_START_EXIT_DISPATCH_PROF();
8312	#ifdef HMVMX_USE_FUNCTION_TABLE
8313	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8314	#else
8315	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8316	#endif
8317	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8318	if (rc != VINF_SUCCESS)
8319	break;
8320	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8321	{
8322	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8323	rc = VINF_EM_RAW_INTERRUPT;
8324	break;
8325	}
8326	}
8327
8328	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8329	return rc;
8330	}
8331
8332
8333	/**
8334	* Single steps guest code using VT-x.
8335	*
8336	* @returns VBox status code.
8337	* @param pVM Pointer to the VM.
8338	* @param pVCpu Pointer to the VMCPU.
8339	* @param pCtx Pointer to the guest-CPU context.
8340	*
8341	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8342	*/
8343	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8344	{
8345	VMXTRANSIENT VmxTransient;
8346	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8347	int rc = VERR_INTERNAL_ERROR_5;
8348	uint32_t cLoops = 0;
8349	uint16_t uCsStart = pCtx->cs.Sel;
8350	uint64_t uRipStart = pCtx->rip;
8351
8352	for (;; cLoops++)
8353	{
8354	Assert(!HMR0SuspendPending());
8355	HMVMX_ASSERT_CPU_SAFE();
8356
8357	/* Preparatory work for running guest code, this may force us to return
8358	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8359	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8360	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8361	if (rc != VINF_SUCCESS)
8362	break;
8363
8364	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8365	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8366	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8367
8368	/* Restore any residual host-state and save any bits shared between host
8369	and guest into the guest-CPU state. Re-enables interrupts! */
8370	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8371
8372	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8373	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8374	{
8375	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8376	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8377	return rc;
8378	}
8379
8380	/* Handle the VM-exit. */
8381	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8383	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8384	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8385	HMVMX_START_EXIT_DISPATCH_PROF();
8386	#ifdef HMVMX_USE_FUNCTION_TABLE
8387	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8388	#else
8389	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8390	#endif
8391	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8392	if (rc != VINF_SUCCESS)
8393	break;
8394	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8395	{
8396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8397	rc = VINF_EM_RAW_INTERRUPT;
8398	break;
8399	}
8400
8401	/*
8402	* Did the RIP change, if so, consider it a single step.
8403	* Otherwise, make sure one of the TFs gets set.
8404	*/
8405	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8406	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8407	AssertRCReturn(rc2, rc2);
8408	if ( pCtx->rip != uRipStart
8409	\|\| pCtx->cs.Sel != uCsStart)
8410	{
8411	rc = VINF_EM_DBG_STEPPED;
8412	break;
8413	}
8414	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8415	}
8416
8417	/*
8418	* Clear the X86_EFL_TF if necessary.
8419	*/
8420	if (pVCpu->hm.s.fClearTrapFlag)
8421	{
8422	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8423	AssertRCReturn(rc2, rc2);
8424	pVCpu->hm.s.fClearTrapFlag = false;
8425	pCtx->eflags.Bits.u1TF = 0;
8426	}
8427	/** @todo there seems to be issues with the resume flag when the monitor trap
8428	* flag is pending without being used. Seen early in bios init when
8429	* accessing APIC page in prot mode. */
8430
8431	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8432	return rc;
8433	}
8434
8435
8436	/**
8437	* Runs the guest code using VT-x.
8438	*
8439	* @returns VBox status code.
8440	* @param pVM Pointer to the VM.
8441	* @param pVCpu Pointer to the VMCPU.
8442	* @param pCtx Pointer to the guest-CPU context.
8443	*/
8444	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8445	{
8446	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8447	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8448	HMVMX_ASSERT_PREEMPT_SAFE();
8449
8450	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8451
8452	int rc;
8453	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8454	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8455	else
8456	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8457
8458	if (rc == VERR_EM_INTERPRETER)
8459	rc = VINF_EM_RAW_EMULATE_INSTR;
8460	else if (rc == VINF_EM_RESET)
8461	rc = VINF_EM_TRIPLE_FAULT;
8462
8463	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8464	if (RT_FAILURE(rc2))
8465	{
8466	pVCpu->hm.s.u32HMError = rc;
8467	rc = rc2;
8468	}
8469	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8470	return rc;
8471	}
8472
8473
8474	#ifndef HMVMX_USE_FUNCTION_TABLE
8475	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8476	{
8477	#ifdef DEBUG_ramshankar
8478	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8479	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8480	#endif
8481	int rc;
8482	switch (rcReason)
8483	{
8484	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8511	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8512	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8513	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8514	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8515	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8516	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8517
8518	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8519	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8520	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8521	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8522	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8523	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8524	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8525	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8526	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8527
8528	case VMX_EXIT_VMCALL:
8529	case VMX_EXIT_VMCLEAR:
8530	case VMX_EXIT_VMLAUNCH:
8531	case VMX_EXIT_VMPTRLD:
8532	case VMX_EXIT_VMPTRST:
8533	case VMX_EXIT_VMREAD:
8534	case VMX_EXIT_VMRESUME:
8535	case VMX_EXIT_VMWRITE:
8536	case VMX_EXIT_VMXOFF:
8537	case VMX_EXIT_VMXON:
8538	case VMX_EXIT_INVEPT:
8539	case VMX_EXIT_INVVPID:
8540	case VMX_EXIT_VMFUNC:
8541	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8542	break;
8543	default:
8544	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8545	break;
8546	}
8547	return rc;
8548	}
8549	#endif
8550
8551	#ifdef DEBUG
8552	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8553	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8554	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8555
8556	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8557	do \
8558	{ \
8559	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8560	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8561	} while (0)
8562
8563	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8564	do { \
8565	AssertPtr(pVCpu); \
8566	AssertPtr(pMixedCtx); \
8567	AssertPtr(pVmxTransient); \
8568	Assert(pVmxTransient->fVMEntryFailed == false); \
8569	Assert(ASMIntAreEnabled()); \
8570	HMVMX_ASSERT_PREEMPT_SAFE(); \
8571	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8572	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8573	HMVMX_ASSERT_PREEMPT_SAFE(); \
8574	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8575	HMVMX_ASSERT_PREEMPT_CPUID(); \
8576	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8577	} while (0)
8578
8579	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8580	do { \
8581	Log4Func(("\n")); \
8582	} while (0)
8583	#else /* Release builds */
8584	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8585	do { \
8586	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8587	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8588	} while (0)
8589	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8590	#endif
8591
8592
8593	/**
8594	* Advances the guest RIP after reading it from the VMCS.
8595	*
8596	* @returns VBox status code.
8597	* @param pVCpu Pointer to the VMCPU.
8598	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8599	* out-of-sync. Make sure to update the required fields
8600	* before using them.
8601	* @param pVmxTransient Pointer to the VMX transient structure.
8602	*
8603	* @remarks No-long-jump zone!!!
8604	*/
8605	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8606	{
8607	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8608	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8609	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8610	AssertRCReturn(rc, rc);
8611
8612	pMixedCtx->rip += pVmxTransient->cbInstr;
8613	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8614
8615	/*
8616	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8617	* pending debug exception field as it takes care of priority of events.
8618	*
8619	* See Intel spec. 32.2.1 "Debug Exceptions".
8620	*/
8621	if (pMixedCtx->eflags.Bits.u1TF)
8622	{
8623	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
8624	AssertRCReturn(rc, rc);
8625	}
8626
8627	return rc;
8628	}
8629
8630
8631	/**
8632	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8633	* and update error record fields accordingly.
8634	*
8635	* @return VMX_IGS_* return codes.
8636	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8637	* wrong with the guest state.
8638	*
8639	* @param pVM Pointer to the VM.
8640	* @param pVCpu Pointer to the VMCPU.
8641	* @param pCtx Pointer to the guest-CPU state.
8642	*/
8643	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8644	{
8645	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8646	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8647	uError = (err); \
8648	break; \
8649	} else do { } while (0)
8650
8651	int rc;
8652	uint32_t uError = VMX_IGS_ERROR;
8653	uint32_t u32Val;
8654	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8655
8656	do
8657	{
8658	/*
8659	* CR0.
8660	*/
8661	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8662	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8663	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8664	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8665	if (fUnrestrictedGuest)
8666	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8667
8668	uint32_t u32GuestCR0;
8669	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8670	AssertRCBreak(rc);
8671	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8672	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8673	if ( !fUnrestrictedGuest
8674	&& (u32GuestCR0 & X86_CR0_PG)
8675	&& !(u32GuestCR0 & X86_CR0_PE))
8676	{
8677	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8678	}
8679
8680	/*
8681	* CR4.
8682	*/
8683	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8684	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8685
8686	uint32_t u32GuestCR4;
8687	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8688	AssertRCBreak(rc);
8689	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8690	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8691
8692	/*
8693	* IA32_DEBUGCTL MSR.
8694	*/
8695	uint64_t u64Val;
8696	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8697	AssertRCBreak(rc);
8698	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8699	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8700	{
8701	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8702	}
8703	uint64_t u64DebugCtlMsr = u64Val;
8704
8705	#ifdef VBOX_STRICT
8706	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8707	AssertRCBreak(rc);
8708	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8709	#endif
8710	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8711
8712	/*
8713	* RIP and RFLAGS.
8714	*/
8715	uint32_t u32Eflags;
8716	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8717	if (HMVMX_IS_64BIT_HOST_MODE())
8718	{
8719	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8720	AssertRCBreak(rc);
8721	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8722	if ( !fLongModeGuest
8723	\|\| !pCtx->cs.Attr.n.u1Long)
8724	{
8725	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8726	}
8727	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8728	* must be identical if the "IA32e mode guest" VM-entry control is 1
8729	* and CS.L is 1. No check applies if the CPU supports 64
8730	* linear-address bits. */
8731
8732	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8733	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8734	AssertRCBreak(rc);
8735	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8736	VMX_IGS_RFLAGS_RESERVED);
8737	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8738	u32Eflags = u64Val;
8739	}
8740	else
8741	#endif
8742	{
8743	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8744	AssertRCBreak(rc);
8745	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8746	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8747	}
8748
8749	if ( fLongModeGuest
8750	\|\| ( fUnrestrictedGuest
8751	&& !(u32GuestCR0 & X86_CR0_PE)))
8752	{
8753	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8754	}
8755
8756	uint32_t u32EntryInfo;
8757	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8758	AssertRCBreak(rc);
8759	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8760	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8761	{
8762	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8763	}
8764
8765	/*
8766	* 64-bit checks.
8767	*/
8768	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8769	if (HMVMX_IS_64BIT_HOST_MODE())
8770	{
8771	if ( fLongModeGuest
8772	&& !fUnrestrictedGuest)
8773	{
8774	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8775	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8776	}
8777
8778	if ( !fLongModeGuest
8779	&& (u32GuestCR4 & X86_CR4_PCIDE))
8780	{
8781	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8782	}
8783
8784	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8785	* 51:32 beyond the processor's physical-address width are 0. */
8786
8787	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8788	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8789	{
8790	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8791	}
8792
8793	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8794	AssertRCBreak(rc);
8795	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8796
8797	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8798	AssertRCBreak(rc);
8799	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8800	}
8801	#endif
8802
8803	/*
8804	* PERF_GLOBAL MSR.
8805	*/
8806	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8807	{
8808	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8809	AssertRCBreak(rc);
8810	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8811	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8812	}
8813
8814	/*
8815	* PAT MSR.
8816	*/
8817	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8818	{
8819	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8820	AssertRCBreak(rc);
8821	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8822	for (unsigned i = 0; i < 8; i++)
8823	{
8824	uint8_t u8Val = (u64Val & 0x7);
8825	if ( u8Val != 0 /* UC */
8826	\|\| u8Val != 1 /* WC */
8827	\|\| u8Val != 4 /* WT */
8828	\|\| u8Val != 5 /* WP */
8829	\|\| u8Val != 6 /* WB */
8830	\|\| u8Val != 7 /* UC- */)
8831	{
8832	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8833	}
8834	u64Val >>= 3;
8835	}
8836	}
8837
8838	/*
8839	* EFER MSR.
8840	*/
8841	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8842	{
8843	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8844	AssertRCBreak(rc);
8845	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8846	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8847	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8848	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8849	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8850	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8851	}
8852
8853	/*
8854	* Segment registers.
8855	*/
8856	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8857	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8858	if (!(u32Eflags & X86_EFL_VM))
8859	{
8860	/* CS */
8861	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8862	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8863	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8864	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8865	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8866	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8867	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8868	/* CS cannot be loaded with NULL in protected mode. */
8869	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8870	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8871	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8872	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8873	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8874	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8875	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8876	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8877	else
8878	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8879
8880	/* SS */
8881	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8882	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8883	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8884	if ( !(pCtx->cr0 & X86_CR0_PE)
8885	\|\| pCtx->cs.Attr.n.u4Type == 3)
8886	{
8887	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8888	}
8889	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8890	{
8891	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8892	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8893	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8894	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8895	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8896	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8897	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8898	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8899	}
8900
8901	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8902	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8903	{
8904	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8905	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8906	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8907	\|\| pCtx->ds.Attr.n.u4Type > 11
8908	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8909	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8910	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8911	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8912	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8913	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8914	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8915	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8916	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8917	}
8918	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8919	{
8920	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8921	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8922	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8923	\|\| pCtx->es.Attr.n.u4Type > 11
8924	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8925	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8926	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8927	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8928	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8929	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8930	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8931	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8932	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8933	}
8934	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8935	{
8936	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8937	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8938	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8939	\|\| pCtx->fs.Attr.n.u4Type > 11
8940	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8941	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8942	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8943	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8944	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8945	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8946	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8947	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8948	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8949	}
8950	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8951	{
8952	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8953	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8954	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8955	\|\| pCtx->gs.Attr.n.u4Type > 11
8956	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8957	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8958	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8959	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8960	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8961	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8962	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8963	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8964	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8965	}
8966	/* 64-bit capable CPUs. */
8967	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8968	if (HMVMX_IS_64BIT_HOST_MODE())
8969	{
8970	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8971	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8972	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8973	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8974	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8975	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8976	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8977	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8978	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8979	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8980	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8981	}
8982	#endif
8983	}
8984	else
8985	{
8986	/* V86 mode checks. */
8987	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8988	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8989	{
8990	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8991	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8992	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8993	}
8994	else
8995	{
8996	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8997	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8998	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8999	}
9000
9001	/* CS */
9002	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9003	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9004	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9005	/* SS */
9006	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9007	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9008	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9009	/* DS */
9010	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9011	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9012	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9013	/* ES */
9014	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9015	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9016	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9017	/* FS */
9018	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9019	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9020	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9021	/* GS */
9022	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9023	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9024	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9025	/* 64-bit capable CPUs. */
9026	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9027	if (HMVMX_IS_64BIT_HOST_MODE())
9028	{
9029	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9030	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9031	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9032	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9033	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9034	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9035	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9036	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9037	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9038	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9039	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9040	}
9041	#endif
9042	}
9043
9044	/*
9045	* TR.
9046	*/
9047	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9048	/* 64-bit capable CPUs. */
9049	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9050	if (HMVMX_IS_64BIT_HOST_MODE())
9051	{
9052	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9053	}
9054	#endif
9055	if (fLongModeGuest)
9056	{
9057	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9058	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9059	}
9060	else
9061	{
9062	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9063	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9064	VMX_IGS_TR_ATTR_TYPE_INVALID);
9065	}
9066	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9067	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9068	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9069	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9070	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9071	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9072	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9073	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9074
9075	/*
9076	* GDTR and IDTR.
9077	*/
9078	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9079	if (HMVMX_IS_64BIT_HOST_MODE())
9080	{
9081	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9082	AssertRCBreak(rc);
9083	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9084
9085	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9086	AssertRCBreak(rc);
9087	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9088	}
9089	#endif
9090
9091	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9092	AssertRCBreak(rc);
9093	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9094
9095	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9096	AssertRCBreak(rc);
9097	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9098
9099	/*
9100	* Guest Non-Register State.
9101	*/
9102	/* Activity State. */
9103	uint32_t u32ActivityState;
9104	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9105	AssertRCBreak(rc);
9106	HMVMX_CHECK_BREAK( !u32ActivityState
9107	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9108	VMX_IGS_ACTIVITY_STATE_INVALID);
9109	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9110	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9111	uint32_t u32IntrState;
9112	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9113	AssertRCBreak(rc);
9114	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9115	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9116	{
9117	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9118	}
9119
9120	/** @todo Activity state and injecting interrupts. Left as a todo since we
9121	* currently don't use activity states but ACTIVE. */
9122
9123	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9124	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9125
9126	/* Guest interruptibility-state. */
9127	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9128	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9129	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9130	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9131	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9132	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9133	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9134	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9135	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9136	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9137	{
9138	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9139	{
9140	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9141	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9142	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9143	}
9144	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9145	{
9146	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9147	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9148	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9149	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9150	}
9151	}
9152	/** @todo Assumes the processor is not in SMM. */
9153	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9154	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9155	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9156	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9157	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9158	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9159	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9160	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9161	{
9162	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9163	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9164	}
9165
9166	/* Pending debug exceptions. */
9167	if (HMVMX_IS_64BIT_HOST_MODE())
9168	{
9169	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9170	AssertRCBreak(rc);
9171	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9172	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9173	u32Val = u64Val; /* For pending debug exceptions checks below. */
9174	}
9175	else
9176	{
9177	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9178	AssertRCBreak(rc);
9179	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9180	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9181	}
9182
9183	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9184	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9185	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9186	{
9187	if ( (u32Eflags & X86_EFL_TF)
9188	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9189	{
9190	/* Bit 14 is PendingDebug.BS. */
9191	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9192	}
9193	if ( !(u32Eflags & X86_EFL_TF)
9194	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9195	{
9196	/* Bit 14 is PendingDebug.BS. */
9197	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9198	}
9199	}
9200
9201	/* VMCS link pointer. */
9202	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9203	AssertRCBreak(rc);
9204	if (u64Val != UINT64_C(0xffffffffffffffff))
9205	{
9206	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9207	/** @todo Bits beyond the processor's physical-address width MBZ. */
9208	/** @todo 32-bit located in memory referenced by value of this field (as a
9209	* physical address) must contain the processor's VMCS revision ID. */
9210	/** @todo SMM checks. */
9211	}
9212
9213	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9214	* not using Nested Paging? */
9215	if ( pVM->hm.s.fNestedPaging
9216	&& !fLongModeGuest
9217	&& CPUMIsGuestInPAEModeEx(pCtx))
9218	{
9219	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9220	AssertRCBreak(rc);
9221	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9222
9223	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9224	AssertRCBreak(rc);
9225	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9226
9227	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9228	AssertRCBreak(rc);
9229	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9230
9231	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9232	AssertRCBreak(rc);
9233	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9234	}
9235
9236	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9237	if (uError == VMX_IGS_ERROR)
9238	uError = VMX_IGS_REASON_NOT_FOUND;
9239	} while (0);
9240
9241	pVCpu->hm.s.u32HMError = uError;
9242	return uError;
9243
9244	#undef HMVMX_ERROR_BREAK
9245	#undef HMVMX_CHECK_BREAK
9246	}
9247
9248	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9249	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9250	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9251
9252	/** @name VM-exit handlers.
9253	* @{
9254	*/
9255
9256	/**
9257	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9258	*/
9259	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9260	{
9261	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9263	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9264	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9265	return VINF_SUCCESS;
9266	return VINF_EM_RAW_INTERRUPT;
9267	}
9268
9269
9270	/**
9271	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9272	*/
9273	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9274	{
9275	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9276	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9277
9278	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9279	AssertRCReturn(rc, rc);
9280
9281	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9282	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9283	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9284	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9285
9286	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9287	{
9288	/*
9289	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9290	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9291	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9292	*
9293	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9294	*/
9295	VMXDispatchHostNmi();
9296	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9297	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9298	return VINF_SUCCESS;
9299	}
9300
9301	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9302	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9303	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9304	{
9305	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9306	return VINF_SUCCESS;
9307	}
9308	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9309	{
9310	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9311	return rc;
9312	}
9313
9314	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9315	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9316	switch (uIntType)
9317	{
9318	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9319	Assert(uVector == X86_XCPT_DB);
9320	/* no break */
9321	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9322	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9323	/* no break */
9324	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9325	{
9326	switch (uVector)
9327	{
9328	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9329	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9330	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9331	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9332	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9333	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9334	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9335	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9336	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9337	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9338	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9339	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9340	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9341	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9342	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9343	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9344	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9345	#endif
9346	default:
9347	{
9348	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9349	AssertRCReturn(rc, rc);
9350
9351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9352	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9353	{
9354	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9355	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9356	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9357
9358	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9359	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9360	AssertRCReturn(rc, rc);
9361	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9362	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9363	0 /* GCPtrFaultAddress */);
9364	AssertRCReturn(rc, rc);
9365	}
9366	else
9367	{
9368	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9369	pVCpu->hm.s.u32HMError = uVector;
9370	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9371	}
9372	break;
9373	}
9374	}
9375	break;
9376	}
9377
9378	default:
9379	{
9380	pVCpu->hm.s.u32HMError = uExitIntInfo;
9381	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9382	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9383	break;
9384	}
9385	}
9386	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9387	return rc;
9388	}
9389
9390
9391	/**
9392	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9393	*/
9394	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9395	{
9396	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9397
9398	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9399	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9400	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9401	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9402	AssertRCReturn(rc, rc);
9403
9404	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9405	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9406	return VINF_SUCCESS;
9407	}
9408
9409
9410	/**
9411	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9412	*/
9413	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9414	{
9415	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9416	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9417	HMVMX_RETURN_UNEXPECTED_EXIT();
9418	}
9419
9420
9421	/**
9422	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9423	*/
9424	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9425	{
9426	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9427	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9428	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9429	}
9430
9431
9432	/**
9433	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9434	*/
9435	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9436	{
9437	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9439	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9440	}
9441
9442
9443	/**
9444	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9445	*/
9446	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9447	{
9448	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9449	PVM pVM = pVCpu->CTX_SUFF(pVM);
9450	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9451	if (RT_LIKELY(rc == VINF_SUCCESS))
9452	{
9453	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9454	Assert(pVmxTransient->cbInstr == 2);
9455	}
9456	else
9457	{
9458	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9459	rc = VERR_EM_INTERPRETER;
9460	}
9461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9462	return rc;
9463	}
9464
9465
9466	/**
9467	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9468	*/
9469	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9470	{
9471	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9472	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9473	AssertRCReturn(rc, rc);
9474
9475	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9476	return VINF_EM_RAW_EMULATE_INSTR;
9477
9478	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9479	HMVMX_RETURN_UNEXPECTED_EXIT();
9480	}
9481
9482
9483	/**
9484	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9485	*/
9486	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9487	{
9488	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9489	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9490	AssertRCReturn(rc, rc);
9491
9492	PVM pVM = pVCpu->CTX_SUFF(pVM);
9493	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9494	if (RT_LIKELY(rc == VINF_SUCCESS))
9495	{
9496	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9497	Assert(pVmxTransient->cbInstr == 2);
9498	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9499	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9500	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9501	}
9502	else
9503	{
9504	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9505	rc = VERR_EM_INTERPRETER;
9506	}
9507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9508	return rc;
9509	}
9510
9511
9512	/**
9513	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9514	*/
9515	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9516	{
9517	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9518	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9519	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9520	AssertRCReturn(rc, rc);
9521
9522	PVM pVM = pVCpu->CTX_SUFF(pVM);
9523	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9524	if (RT_LIKELY(rc == VINF_SUCCESS))
9525	{
9526	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9527	Assert(pVmxTransient->cbInstr == 3);
9528	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9529	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9530	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9531	}
9532	else
9533	{
9534	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9535	rc = VERR_EM_INTERPRETER;
9536	}
9537	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9538	return rc;
9539	}
9540
9541
9542	/**
9543	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9544	*/
9545	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9546	{
9547	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9548	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9549	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9550	AssertRCReturn(rc, rc);
9551
9552	PVM pVM = pVCpu->CTX_SUFF(pVM);
9553	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9554	if (RT_LIKELY(rc == VINF_SUCCESS))
9555	{
9556	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9557	Assert(pVmxTransient->cbInstr == 2);
9558	}
9559	else
9560	{
9561	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9562	rc = VERR_EM_INTERPRETER;
9563	}
9564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9565	return rc;
9566	}
9567
9568
9569	/**
9570	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9571	*/
9572	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9573	{
9574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9575	PVM pVM = pVCpu->CTX_SUFF(pVM);
9576	Assert(!pVM->hm.s.fNestedPaging);
9577
9578	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9579	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9580	AssertRCReturn(rc, rc);
9581
9582	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9583	rc = VBOXSTRICTRC_VAL(rc2);
9584	if (RT_LIKELY(rc == VINF_SUCCESS))
9585	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9586	else
9587	{
9588	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9589	pVmxTransient->uExitQualification, rc));
9590	}
9591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9592	return rc;
9593	}
9594
9595
9596	/**
9597	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9598	*/
9599	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9600	{
9601	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9602	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9603	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9604	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9605	AssertRCReturn(rc, rc);
9606
9607	PVM pVM = pVCpu->CTX_SUFF(pVM);
9608	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9609	if (RT_LIKELY(rc == VINF_SUCCESS))
9610	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9611	else
9612	{
9613	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9614	rc = VERR_EM_INTERPRETER;
9615	}
9616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9617	return rc;
9618	}
9619
9620
9621	/**
9622	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9623	*/
9624	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9625	{
9626	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9627	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9628	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9629	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9630	AssertRCReturn(rc, rc);
9631
9632	PVM pVM = pVCpu->CTX_SUFF(pVM);
9633	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9634	rc = VBOXSTRICTRC_VAL(rc2);
9635	if (RT_LIKELY( rc == VINF_SUCCESS
9636	\|\| rc == VINF_EM_HALT))
9637	{
9638	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9639	AssertRCReturn(rc3, rc3);
9640
9641	if ( rc == VINF_EM_HALT
9642	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9643	{
9644	rc = VINF_SUCCESS;
9645	}
9646	}
9647	else
9648	{
9649	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9650	rc = VERR_EM_INTERPRETER;
9651	}
9652	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9653	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9654	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9655	return rc;
9656	}
9657
9658
9659	/**
9660	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9661	*/
9662	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9663	{
9664	/*
9665	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9666	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9667	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9668	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9669	*/
9670	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9671	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9672	HMVMX_RETURN_UNEXPECTED_EXIT();
9673	}
9674
9675
9676	/**
9677	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9678	*/
9679	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9680	{
9681	/*
9682	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9683	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9684	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9685	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9686	*/
9687	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9688	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9689	HMVMX_RETURN_UNEXPECTED_EXIT();
9690	}
9691
9692
9693	/**
9694	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9695	*/
9696	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9697	{
9698	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9699	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9700	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9701	HMVMX_RETURN_UNEXPECTED_EXIT();
9702	}
9703
9704
9705	/**
9706	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9707	*/
9708	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9709	{
9710	/*
9711	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9712	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9713	* See Intel spec. 25.3 "Other Causes of VM-exits".
9714	*/
9715	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9716	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9717	HMVMX_RETURN_UNEXPECTED_EXIT();
9718	}
9719
9720
9721	/**
9722	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9723	* VM-exit.
9724	*/
9725	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9726	{
9727	/*
9728	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9729	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9730	*
9731	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9732	* See Intel spec. "23.8 Restrictions on VMX operation".
9733	*/
9734	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9735	return VINF_SUCCESS;
9736	}
9737
9738
9739	/**
9740	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9741	* VM-exit.
9742	*/
9743	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9744	{
9745	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9746	return VINF_EM_RESET;
9747	}
9748
9749
9750	/**
9751	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9752	*/
9753	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9754	{
9755	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9756	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9757	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9758	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9759	AssertRCReturn(rc, rc);
9760
9761	pMixedCtx->rip++;
9762	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9763	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9764	rc = VINF_SUCCESS;
9765	else
9766	rc = VINF_EM_HALT;
9767
9768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9769	return rc;
9770	}
9771
9772
9773	/**
9774	* VM-exit handler for instructions that result in a #UD exception delivered to
9775	* the guest.
9776	*/
9777	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9778	{
9779	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9780	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9781	return VINF_SUCCESS;
9782	}
9783
9784
9785	/**
9786	* VM-exit handler for expiry of the VMX preemption timer.
9787	*/
9788	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9789	{
9790	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9791
9792	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9793	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9794
9795	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9796	PVM pVM = pVCpu->CTX_SUFF(pVM);
9797	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9798	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9799	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9800	}
9801
9802
9803	/**
9804	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9805	*/
9806	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9807	{
9808	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9809
9810	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9811	/** @todo check if XSETBV is supported by the recompiler. */
9812	return VERR_EM_INTERPRETER;
9813	}
9814
9815
9816	/**
9817	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9818	*/
9819	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9820	{
9821	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9822
9823	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9824	/** @todo implement EMInterpretInvpcid() */
9825	return VERR_EM_INTERPRETER;
9826	}
9827
9828
9829	/**
9830	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9831	* Error VM-exit.
9832	*/
9833	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9834	{
9835	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9836	AssertRCReturn(rc, rc);
9837
9838	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9839	NOREF(uInvalidReason);
9840
9841	#ifdef VBOX_STRICT
9842	uint32_t uIntrState;
9843	HMVMXHCUINTREG uHCReg;
9844	uint64_t u64Val;
9845	uint32_t u32Val;
9846
9847	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9848	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9849	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9850	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9851	AssertRCReturn(rc, rc);
9852
9853	Log4(("uInvalidReason %u\n", uInvalidReason));
9854	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9855	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9856	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9857	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9858
9859	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9860	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9861	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9862	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9863	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9864	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9865	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9866	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9867	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9868	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9869	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9870	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9871	#else
9872	NOREF(pVmxTransient);
9873	#endif
9874
9875	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9876	return VERR_VMX_INVALID_GUEST_STATE;
9877	}
9878
9879
9880	/**
9881	* VM-exit handler for VM-entry failure due to an MSR-load
9882	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9883	*/
9884	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9885	{
9886	NOREF(pVmxTransient);
9887	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9888	HMVMX_RETURN_UNEXPECTED_EXIT();
9889	}
9890
9891
9892	/**
9893	* VM-exit handler for VM-entry failure due to a machine-check event
9894	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9895	*/
9896	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9897	{
9898	NOREF(pVmxTransient);
9899	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9900	HMVMX_RETURN_UNEXPECTED_EXIT();
9901	}
9902
9903
9904	/**
9905	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9906	* theory.
9907	*/
9908	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9909	{
9910	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9911	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9912	return VERR_VMX_UNDEFINED_EXIT_CODE;
9913	}
9914
9915
9916	/**
9917	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9918	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9919	* Conditional VM-exit.
9920	*/
9921	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9922	{
9923	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9924
9925	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9927	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9928	return VERR_EM_INTERPRETER;
9929	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9930	HMVMX_RETURN_UNEXPECTED_EXIT();
9931	}
9932
9933
9934	/**
9935	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9936	*/
9937	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9938	{
9939	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9940
9941	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9943	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9944	return VERR_EM_INTERPRETER;
9945	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9946	HMVMX_RETURN_UNEXPECTED_EXIT();
9947	}
9948
9949
9950	/**
9951	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9952	*/
9953	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9954	{
9955	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9956
9957	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9958	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9959	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9960	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9961	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9962	{
9963	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9964	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9965	}
9966	AssertRCReturn(rc, rc);
9967	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9968
9969	#ifdef VBOX_STRICT
9970	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9971	{
9972	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9973	{
9974	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9975	HMVMX_RETURN_UNEXPECTED_EXIT();
9976	}
9977	# if HC_ARCH_BITS == 64
9978	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9979	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9980	{
9981	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9982	HMVMX_RETURN_UNEXPECTED_EXIT();
9983	}
9984	# endif
9985	}
9986	#endif
9987
9988	PVM pVM = pVCpu->CTX_SUFF(pVM);
9989	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9990	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9991	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9993
9994	if (RT_LIKELY(rc == VINF_SUCCESS))
9995	{
9996	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9997	Assert(pVmxTransient->cbInstr == 2);
9998	}
9999	return rc;
10000	}
10001
10002
10003	/**
10004	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10005	*/
10006	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10007	{
10008	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10009	PVM pVM = pVCpu->CTX_SUFF(pVM);
10010	int rc = VINF_SUCCESS;
10011
10012	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10013	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10014	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10015	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10016	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10017	{
10018	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10019	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10020	}
10021	AssertRCReturn(rc, rc);
10022	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10023
10024	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10025	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10026	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10027
10028	if (RT_LIKELY(rc == VINF_SUCCESS))
10029	{
10030	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10031
10032	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10033	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10034	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10035	{
10036	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10037	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10038	EMInterpretWrmsr() changes it. */
10039	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10040	}
10041	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10042	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10043
10044	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10045	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10046	{
10047	switch (pMixedCtx->ecx)
10048	{
10049	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10050	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10051	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10052	case MSR_K8_FS_BASE: /* no break */
10053	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10054	default:
10055	{
10056	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10057	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10058	#if HC_ARCH_BITS == 64
10059	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10060	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10061	#endif
10062	break;
10063	}
10064	}
10065	}
10066	#ifdef VBOX_STRICT
10067	else
10068	{
10069	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10070	switch (pMixedCtx->ecx)
10071	{
10072	case MSR_IA32_SYSENTER_CS:
10073	case MSR_IA32_SYSENTER_EIP:
10074	case MSR_IA32_SYSENTER_ESP:
10075	case MSR_K8_FS_BASE:
10076	case MSR_K8_GS_BASE:
10077	{
10078	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10079	HMVMX_RETURN_UNEXPECTED_EXIT();
10080	}
10081
10082	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10083	default:
10084	{
10085	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10086	{
10087	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10088	pMixedCtx->ecx));
10089	HMVMX_RETURN_UNEXPECTED_EXIT();
10090	}
10091
10092	#if HC_ARCH_BITS == 64
10093	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10094	{
10095	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10096	HMVMX_RETURN_UNEXPECTED_EXIT();
10097	}
10098	#endif
10099	break;
10100	}
10101	}
10102	}
10103	#endif /* VBOX_STRICT */
10104	}
10105	return rc;
10106	}
10107
10108
10109	/**
10110	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10111	*/
10112	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10113	{
10114	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10115
10116	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10118	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10119	return VERR_EM_INTERPRETER;
10120	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10121	HMVMX_RETURN_UNEXPECTED_EXIT();
10122	}
10123
10124
10125	/**
10126	* VM-exit handler for when the TPR value is lowered below the specified
10127	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10128	*/
10129	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10130	{
10131	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10132	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10133
10134	/*
10135	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10136	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10137	* resume guest execution.
10138	*/
10139	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10141	return VINF_SUCCESS;
10142	}
10143
10144
10145	/**
10146	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10147	* VM-exit.
10148	*
10149	* @retval VINF_SUCCESS when guest execution can continue.
10150	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10151	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10152	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10153	* recompiler.
10154	*/
10155	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10156	{
10157	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10158	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10159	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10160	AssertRCReturn(rc, rc);
10161
10162	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10163	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10164	PVM pVM = pVCpu->CTX_SUFF(pVM);
10165	switch (uAccessType)
10166	{
10167	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10168	{
10169	#if 0
10170	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10171	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10172	#else
10173	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10174	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10175	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10176	#endif
10177	AssertRCReturn(rc, rc);
10178
10179	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10180	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10181	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10182	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10183
10184	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10185	{
10186	case 0: /* CR0 */
10187	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10188	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10189	break;
10190	case 2: /* CR2 */
10191	/* Nothing to do here, CR2 it's not part of the VMCS. */
10192	break;
10193	case 3: /* CR3 */
10194	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10195	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10196	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10197	break;
10198	case 4: /* CR4 */
10199	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10200	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10201	break;
10202	case 8: /* CR8 */
10203	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10204	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10205	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10206	break;
10207	default:
10208	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10209	break;
10210	}
10211
10212	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10213	break;
10214	}
10215
10216	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10217	{
10218	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10219	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10220	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10221	AssertRCReturn(rc, rc);
10222	Assert( !pVM->hm.s.fNestedPaging
10223	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10224	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10225
10226	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10227	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10228	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10229
10230	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10231	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10232	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10233	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10235	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10236	break;
10237	}
10238
10239	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10240	{
10241	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10242	AssertRCReturn(rc, rc);
10243	rc = EMInterpretCLTS(pVM, pVCpu);
10244	AssertRCReturn(rc, rc);
10245	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10247	Log4(("CRX CLTS write rc=%d\n", rc));
10248	break;
10249	}
10250
10251	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10252	{
10253	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10254	AssertRCReturn(rc, rc);
10255	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10256	if (RT_LIKELY(rc == VINF_SUCCESS))
10257	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10258	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10259	Log4(("CRX LMSW write rc=%d\n", rc));
10260	break;
10261	}
10262
10263	default:
10264	{
10265	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10266	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10267	}
10268	}
10269
10270	/* Validate possible error codes. */
10271	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10272	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10273	if (RT_SUCCESS(rc))
10274	{
10275	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10276	AssertRCReturn(rc2, rc2);
10277	}
10278
10279	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10280	return rc;
10281	}
10282
10283
10284	/**
10285	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10286	* VM-exit.
10287	*/
10288	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10289	{
10290	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10291	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10292
10293	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10294	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10295	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10296	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10297	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10298	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10299	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10300	AssertRCReturn(rc2, rc2);
10301
10302	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10303	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10304	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10305	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10306	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10307	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10308	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10309
10310	/* I/O operation lookup arrays. */
10311	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10312	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10313
10314	VBOXSTRICTRC rcStrict;
10315	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10316	const uint32_t cbInstr = pVmxTransient->cbInstr;
10317	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10318	PVM pVM = pVCpu->CTX_SUFF(pVM);
10319	if (fIOString)
10320	{
10321	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10322	/*
10323	* INS/OUTS - I/O String instruction.
10324	*
10325	* Use instruction-information if available, otherwise fall back on
10326	* interpreting the instruction.
10327	*/
10328	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10329	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10330	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10331	{
10332	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10333	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10334	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10335	AssertRCReturn(rc2, rc2);
10336	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10337	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10338	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10339	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10340	if (fIOWrite)
10341	{
10342	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10343	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10344	}
10345	else
10346	{
10347	/*
10348	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10349	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10350	* See Intel Instruction spec. for "INS".
10351	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10352	*/
10353	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10354	}
10355	}
10356	else
10357	{
10358	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10359	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10360	AssertRCReturn(rc2, rc2);
10361	rcStrict = IEMExecOne(pVCpu);
10362	}
10363	/** @todo IEM needs to be setting these flags somehow. */
10364	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10365	fUpdateRipAlready = true;
10366	#else
10367	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10368	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10369	if (RT_SUCCESS(rcStrict))
10370	{
10371	if (fIOWrite)
10372	{
10373	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10374	(DISCPUMODE)pDis->uAddrMode, cbValue);
10375	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10376	}
10377	else
10378	{
10379	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10380	(DISCPUMODE)pDis->uAddrMode, cbValue);
10381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10382	}
10383	}
10384	else
10385	{
10386	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10387	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10388	}
10389	#endif
10390	}
10391	else
10392	{
10393	/*
10394	* IN/OUT - I/O instruction.
10395	*/
10396	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10397	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10398	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10399	if (fIOWrite)
10400	{
10401	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10402	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10403	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10405	}
10406	else
10407	{
10408	uint32_t u32Result = 0;
10409	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10410	if (IOM_SUCCESS(rcStrict))
10411	{
10412	/* Save result of I/O IN instr. in AL/AX/EAX. */
10413	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10414	}
10415	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10416	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10418	}
10419	}
10420
10421	if (IOM_SUCCESS(rcStrict))
10422	{
10423	if (!fUpdateRipAlready)
10424	{
10425	pMixedCtx->rip += cbInstr;
10426	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10427	}
10428
10429	/*
10430	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10431	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10432	*/
10433	if (fIOString)
10434	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10435
10436	/*
10437	* If any I/O breakpoints are armed, we need to check if one triggered
10438	* and take appropriate action.
10439	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10440	*/
10441	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10442	AssertRCReturn(rc2, rc2);
10443
10444	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10445	* execution engines about whether hyper BPs and such are pending. */
10446	uint32_t const uDr7 = pMixedCtx->dr[7];
10447	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10448	&& X86_DR7_ANY_RW_IO(uDr7)
10449	&& (pMixedCtx->cr4 & X86_CR4_DE))
10450	\|\| DBGFBpIsHwIoArmed(pVM)))
10451	{
10452	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10453
10454	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10455	VMMRZCallRing3Disable(pVCpu);
10456	HM_DISABLE_PREEMPT_IF_NEEDED();
10457
10458	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10459
10460	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10461	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10462	{
10463	/* Raise #DB. */
10464	if (fIsGuestDbgActive)
10465	ASMSetDR6(pMixedCtx->dr[6]);
10466	if (pMixedCtx->dr[7] != uDr7)
10467	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10468
10469	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10470	}
10471	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10472	else if ( rcStrict2 != VINF_SUCCESS
10473	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10474	rcStrict = rcStrict2;
10475
10476	HM_RESTORE_PREEMPT_IF_NEEDED();
10477	VMMRZCallRing3Enable(pVCpu);
10478	}
10479	}
10480
10481	#ifdef DEBUG
10482	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10483	Assert(!fIOWrite);
10484	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10485	Assert(fIOWrite);
10486	else
10487	{
10488	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10489	* statuses, that the VMM device and some others may return. See
10490	* IOM_SUCCESS() for guidance. */
10491	AssertMsg( RT_FAILURE(rcStrict)
10492	\|\| rcStrict == VINF_SUCCESS
10493	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10494	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10495	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10496	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10497	}
10498	#endif
10499
10500	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10501	return VBOXSTRICTRC_TODO(rcStrict);
10502	}
10503
10504
10505	/**
10506	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10507	* VM-exit.
10508	*/
10509	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10510	{
10511	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10512
10513	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10514	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10515	AssertRCReturn(rc, rc);
10516	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10517	{
10518	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10519	AssertRCReturn(rc, rc);
10520	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10521	{
10522	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10523
10524	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10525	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10526	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10527	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10528	{
10529	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10530	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10531
10532	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10533	Assert(!pVCpu->hm.s.Event.fPending);
10534	pVCpu->hm.s.Event.fPending = true;
10535	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10536	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10537	AssertRCReturn(rc, rc);
10538	if (fErrorCodeValid)
10539	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10540	else
10541	pVCpu->hm.s.Event.u32ErrCode = 0;
10542	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10543	&& uVector == X86_XCPT_PF)
10544	{
10545	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10546	}
10547
10548	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10549	}
10550	}
10551	}
10552
10553	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10554	* emulation. */
10555	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10556	return VERR_EM_INTERPRETER;
10557	}
10558
10559
10560	/**
10561	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10562	*/
10563	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10564	{
10565	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10566	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10567	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10568	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10569	AssertRCReturn(rc, rc);
10570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10571	return VINF_EM_DBG_STEPPED;
10572	}
10573
10574
10575	/**
10576	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10577	*/
10578	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10579	{
10580	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10581
10582	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10583	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10584	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10585	return VINF_SUCCESS;
10586	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10587	return rc;
10588
10589	#if 0
10590	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10591	* just sync the whole thing. */
10592	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10593	#else
10594	/* Aggressive state sync. for now. */
10595	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10596	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10597	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10598	#endif
10599	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10600	AssertRCReturn(rc, rc);
10601
10602	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10603	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10604	switch (uAccessType)
10605	{
10606	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10607	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10608	{
10609	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10610	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10611	{
10612	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10613	}
10614
10615	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10616	GCPhys &= PAGE_BASE_GC_MASK;
10617	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10618	PVM pVM = pVCpu->CTX_SUFF(pVM);
10619	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10620	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10621
10622	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10623	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10624	CPUMCTX2CORE(pMixedCtx), GCPhys);
10625	rc = VBOXSTRICTRC_VAL(rc2);
10626	Log4(("ApicAccess rc=%d\n", rc));
10627	if ( rc == VINF_SUCCESS
10628	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10629	\|\| rc == VERR_PAGE_NOT_PRESENT)
10630	{
10631	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10632	\| HM_CHANGED_GUEST_RSP
10633	\| HM_CHANGED_GUEST_RFLAGS
10634	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10635	rc = VINF_SUCCESS;
10636	}
10637	break;
10638	}
10639
10640	default:
10641	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10642	rc = VINF_EM_RAW_EMULATE_INSTR;
10643	break;
10644	}
10645
10646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10647	return rc;
10648	}
10649
10650
10651	/**
10652	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10653	* VM-exit.
10654	*/
10655	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10656	{
10657	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10658
10659	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10660	if (pVmxTransient->fWasGuestDebugStateActive)
10661	{
10662	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10663	HMVMX_RETURN_UNEXPECTED_EXIT();
10664	}
10665
10666	int rc = VERR_INTERNAL_ERROR_5;
10667	if ( !DBGFIsStepping(pVCpu)
10668	&& !pVCpu->hm.s.fSingleInstruction
10669	&& !pVmxTransient->fWasHyperDebugStateActive)
10670	{
10671	/* Don't intercept MOV DRx and #DB any more. */
10672	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10673	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10674	AssertRCReturn(rc, rc);
10675
10676	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10677	{
10678	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10679	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10680	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10681	AssertRCReturn(rc, rc);
10682	#endif
10683	}
10684
10685	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10686	VMMRZCallRing3Disable(pVCpu);
10687	HM_DISABLE_PREEMPT_IF_NEEDED();
10688
10689	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10690	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10691	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10692
10693	HM_RESTORE_PREEMPT_IF_NEEDED();
10694	VMMRZCallRing3Enable(pVCpu);
10695
10696	#ifdef VBOX_WITH_STATISTICS
10697	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10698	AssertRCReturn(rc, rc);
10699	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10700	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10701	else
10702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10703	#endif
10704	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10705	return VINF_SUCCESS;
10706	}
10707
10708	/*
10709	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10710	* Update the segment registers and DR7 from the CPU.
10711	*/
10712	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10713	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10714	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10715	AssertRCReturn(rc, rc);
10716	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10717
10718	PVM pVM = pVCpu->CTX_SUFF(pVM);
10719	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10720	{
10721	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10722	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10723	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10724	if (RT_SUCCESS(rc))
10725	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10727	}
10728	else
10729	{
10730	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10731	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10732	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10733	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10734	}
10735
10736	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10737	if (RT_SUCCESS(rc))
10738	{
10739	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10740	AssertRCReturn(rc2, rc2);
10741	}
10742	return rc;
10743	}
10744
10745
10746	/**
10747	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10748	* Conditional VM-exit.
10749	*/
10750	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10751	{
10752	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10753	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10754
10755	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10756	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10757	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10758	return VINF_SUCCESS;
10759	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10760	return rc;
10761
10762	RTGCPHYS GCPhys = 0;
10763	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10764
10765	#if 0
10766	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10767	#else
10768	/* Aggressive state sync. for now. */
10769	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10770	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10771	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10772	#endif
10773	AssertRCReturn(rc, rc);
10774
10775	/*
10776	* If we succeed, resume guest execution.
10777	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10778	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10779	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10780	* weird case. See @bugref{6043}.
10781	*/
10782	PVM pVM = pVCpu->CTX_SUFF(pVM);
10783	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10784	rc = VBOXSTRICTRC_VAL(rc2);
10785	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10786	if ( rc == VINF_SUCCESS
10787	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10788	\|\| rc == VERR_PAGE_NOT_PRESENT)
10789	{
10790	/* Successfully handled MMIO operation. */
10791	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10792	\| HM_CHANGED_GUEST_RSP
10793	\| HM_CHANGED_GUEST_RFLAGS
10794	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10795	rc = VINF_SUCCESS;
10796	}
10797	return rc;
10798	}
10799
10800
10801	/**
10802	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10803	* VM-exit.
10804	*/
10805	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10806	{
10807	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10808	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10809
10810	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10811	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10812	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10813	return VINF_SUCCESS;
10814	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10815	return rc;
10816
10817	RTGCPHYS GCPhys = 0;
10818	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10819	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10820	#if 0
10821	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10822	#else
10823	/* Aggressive state sync. for now. */
10824	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10825	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10826	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10827	#endif
10828	AssertRCReturn(rc, rc);
10829
10830	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10831	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10832
10833	RTGCUINT uErrorCode = 0;
10834	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10835	uErrorCode \|= X86_TRAP_PF_ID;
10836	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10837	uErrorCode \|= X86_TRAP_PF_RW;
10838	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10839	uErrorCode \|= X86_TRAP_PF_P;
10840
10841	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10842
10843	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10844	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10845
10846	/* Handle the pagefault trap for the nested shadow table. */
10847	PVM pVM = pVCpu->CTX_SUFF(pVM);
10848	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10849	TRPMResetTrap(pVCpu);
10850
10851	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10852	if ( rc == VINF_SUCCESS
10853	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10854	\|\| rc == VERR_PAGE_NOT_PRESENT)
10855	{
10856	/* Successfully synced our nested page tables. */
10857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10858	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10859	\| HM_CHANGED_GUEST_RSP
10860	\| HM_CHANGED_GUEST_RFLAGS);
10861	return VINF_SUCCESS;
10862	}
10863
10864	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10865	return rc;
10866	}
10867
10868	/** @} */
10869
10870	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10871	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10872	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10873
10874	/** @name VM-exit exception handlers.
10875	* @{
10876	*/
10877
10878	/**
10879	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10880	*/
10881	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10882	{
10883	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10885
10886	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10887	AssertRCReturn(rc, rc);
10888
10889	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10890	{
10891	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10892	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10893
10894	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
10895	* provides VM-exit instruction length. If this causes problem later,
10896	* disassemble the instruction like it's done on AMD-V. */
10897	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10898	AssertRCReturn(rc2, rc2);
10899	return rc;
10900	}
10901
10902	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10903	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10904	return rc;
10905	}
10906
10907
10908	/**
10909	* VM-exit exception handler for #BP (Breakpoint exception).
10910	*/
10911	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10912	{
10913	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10915
10916	/** @todo Try optimize this by not saving the entire guest state unless
10917	* really needed. */
10918	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10919	AssertRCReturn(rc, rc);
10920
10921	PVM pVM = pVCpu->CTX_SUFF(pVM);
10922	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10923	if (rc == VINF_EM_RAW_GUEST_TRAP)
10924	{
10925	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10926	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10927	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10928	AssertRCReturn(rc, rc);
10929
10930	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10931	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10932	}
10933
10934	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10935	return rc;
10936	}
10937
10938
10939	/**
10940	* VM-exit exception handler for #DB (Debug exception).
10941	*/
10942	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10943	{
10944	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10945	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10946	Log6(("XcptDB\n"));
10947
10948	/*
10949	* Get the DR6-like values from the exit qualification and pass it to DBGF
10950	* for processing.
10951	*/
10952	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10953	AssertRCReturn(rc, rc);
10954
10955	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10956	uint64_t uDR6 = X86_DR6_INIT_VAL;
10957	uDR6 \|= ( pVmxTransient->uExitQualification
10958	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10959
10960	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10961	if (rc == VINF_EM_RAW_GUEST_TRAP)
10962	{
10963	/*
10964	* The exception was for the guest. Update DR6, DR7.GD and
10965	* IA32_DEBUGCTL.LBR before forwarding it.
10966	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10967	*/
10968	VMMRZCallRing3Disable(pVCpu);
10969	HM_DISABLE_PREEMPT_IF_NEEDED();
10970
10971	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10972	pMixedCtx->dr[6] \|= uDR6;
10973	if (CPUMIsGuestDebugStateActive(pVCpu))
10974	ASMSetDR6(pMixedCtx->dr[6]);
10975
10976	HM_RESTORE_PREEMPT_IF_NEEDED();
10977	VMMRZCallRing3Enable(pVCpu);
10978
10979	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10980	AssertRCReturn(rc, rc);
10981
10982	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10983	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10984
10985	/* Paranoia. */
10986	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10987	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10988
10989	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10990	AssertRCReturn(rc, rc);
10991
10992	/*
10993	* Raise #DB in the guest.
10994	*
10995	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
10996	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
10997	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
10998	*
10999	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11000	*/
11001	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11002	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11003	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11004	AssertRCReturn(rc, rc);
11005	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11006	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11007	return VINF_SUCCESS;
11008	}
11009
11010	/*
11011	* Not a guest trap, must be a hypervisor related debug event then.
11012	* Update DR6 in case someone is interested in it.
11013	*/
11014	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11015	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11016	CPUMSetHyperDR6(pVCpu, uDR6);
11017
11018	return rc;
11019	}
11020
11021
11022	/**
11023	* VM-exit exception handler for #NM (Device-not-available exception: floating
11024	* point exception).
11025	*/
11026	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11027	{
11028	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11029
11030	/* We require CR0 and EFER. EFER is always up-to-date. */
11031	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11032	AssertRCReturn(rc, rc);
11033
11034	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11035	VMMRZCallRing3Disable(pVCpu);
11036	HM_DISABLE_PREEMPT_IF_NEEDED();
11037
11038	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11039	if (pVmxTransient->fWasGuestFPUStateActive)
11040	{
11041	rc = VINF_EM_RAW_GUEST_TRAP;
11042	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11043	}
11044	else
11045	{
11046	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11047	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11048	#endif
11049	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11050	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11051	}
11052
11053	HM_RESTORE_PREEMPT_IF_NEEDED();
11054	VMMRZCallRing3Enable(pVCpu);
11055
11056	if (rc == VINF_SUCCESS)
11057	{
11058	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11059	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11060	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11061	pVCpu->hm.s.fUseGuestFpu = true;
11062	}
11063	else
11064	{
11065	/* Forward #NM to the guest. */
11066	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11067	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11068	AssertRCReturn(rc, rc);
11069	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11070	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11071	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11072	}
11073
11074	return VINF_SUCCESS;
11075	}
11076
11077
11078	/**
11079	* VM-exit exception handler for #GP (General-protection exception).
11080	*
11081	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11082	*/
11083	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11084	{
11085	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11087
11088	int rc = VERR_INTERNAL_ERROR_5;
11089	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11090	{
11091	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11092	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11093	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11094	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11095	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11096	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11097	AssertRCReturn(rc, rc);
11098	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11099	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11100	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11101	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11102	return rc;
11103	#else
11104	/* We don't intercept #GP. */
11105	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11106	NOREF(pVmxTransient);
11107	return VERR_VMX_UNEXPECTED_EXCEPTION;
11108	#endif
11109	}
11110
11111	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11112	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11113
11114	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11115	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11116	AssertRCReturn(rc, rc);
11117
11118	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11119	uint32_t cbOp = 0;
11120	PVM pVM = pVCpu->CTX_SUFF(pVM);
11121	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11122	if (RT_SUCCESS(rc))
11123	{
11124	rc = VINF_SUCCESS;
11125	Assert(cbOp == pDis->cbInstr);
11126	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11127	switch (pDis->pCurInstr->uOpcode)
11128	{
11129	case OP_CLI:
11130	{
11131	pMixedCtx->eflags.Bits.u1IF = 0;
11132	pMixedCtx->rip += pDis->cbInstr;
11133	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11135	break;
11136	}
11137
11138	case OP_STI:
11139	{
11140	pMixedCtx->eflags.Bits.u1IF = 1;
11141	pMixedCtx->rip += pDis->cbInstr;
11142	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11143	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11144	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11145	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11146	break;
11147	}
11148
11149	case OP_HLT:
11150	{
11151	rc = VINF_EM_HALT;
11152	pMixedCtx->rip += pDis->cbInstr;
11153	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11155	break;
11156	}
11157
11158	case OP_POPF:
11159	{
11160	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11161	uint32_t cbParm = 0;
11162	uint32_t uMask = 0;
11163	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11164	{
11165	cbParm = 4;
11166	uMask = 0xffffffff;
11167	}
11168	else
11169	{
11170	cbParm = 2;
11171	uMask = 0xffff;
11172	}
11173
11174	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11175	RTGCPTR GCPtrStack = 0;
11176	X86EFLAGS Eflags;
11177	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11178	&GCPtrStack);
11179	if (RT_SUCCESS(rc))
11180	{
11181	Assert(sizeof(Eflags.u32) >= cbParm);
11182	Eflags.u32 = 0;
11183	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11184	}
11185	if (RT_FAILURE(rc))
11186	{
11187	rc = VERR_EM_INTERPRETER;
11188	break;
11189	}
11190	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11191	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11192	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11193	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11194	pMixedCtx->esp += cbParm;
11195	pMixedCtx->esp &= uMask;
11196	pMixedCtx->rip += pDis->cbInstr;
11197
11198	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11199	\| HM_CHANGED_GUEST_RSP
11200	\| HM_CHANGED_GUEST_RFLAGS);
11201	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11202	break;
11203	}
11204
11205	case OP_PUSHF:
11206	{
11207	uint32_t cbParm = 0;
11208	uint32_t uMask = 0;
11209	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11210	{
11211	cbParm = 4;
11212	uMask = 0xffffffff;
11213	}
11214	else
11215	{
11216	cbParm = 2;
11217	uMask = 0xffff;
11218	}
11219
11220	/* Get the stack pointer & push the contents of eflags onto the stack. */
11221	RTGCPTR GCPtrStack = 0;
11222	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11223	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11224	if (RT_FAILURE(rc))
11225	{
11226	rc = VERR_EM_INTERPRETER;
11227	break;
11228	}
11229	X86EFLAGS Eflags = pMixedCtx->eflags;
11230	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11231	Eflags.Bits.u1RF = 0;
11232	Eflags.Bits.u1VM = 0;
11233
11234	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11235	if (RT_FAILURE(rc))
11236	{
11237	rc = VERR_EM_INTERPRETER;
11238	break;
11239	}
11240	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11241	pMixedCtx->esp -= cbParm;
11242	pMixedCtx->esp &= uMask;
11243	pMixedCtx->rip += pDis->cbInstr;
11244	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11246	break;
11247	}
11248
11249	case OP_IRET:
11250	{
11251	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11252	* instruction reference. */
11253	RTGCPTR GCPtrStack = 0;
11254	uint32_t uMask = 0xffff;
11255	uint16_t aIretFrame[3];
11256	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11257	{
11258	rc = VERR_EM_INTERPRETER;
11259	break;
11260	}
11261	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11262	&GCPtrStack);
11263	if (RT_SUCCESS(rc))
11264	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11265	if (RT_FAILURE(rc))
11266	{
11267	rc = VERR_EM_INTERPRETER;
11268	break;
11269	}
11270	pMixedCtx->eip = 0;
11271	pMixedCtx->ip = aIretFrame[0];
11272	pMixedCtx->cs.Sel = aIretFrame[1];
11273	pMixedCtx->cs.ValidSel = aIretFrame[1];
11274	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11275	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11276	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11277	pMixedCtx->sp += sizeof(aIretFrame);
11278	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11279	\| HM_CHANGED_GUEST_SEGMENT_REGS
11280	\| HM_CHANGED_GUEST_RSP
11281	\| HM_CHANGED_GUEST_RFLAGS);
11282	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11284	break;
11285	}
11286
11287	case OP_INT:
11288	{
11289	uint16_t uVector = pDis->Param1.uValue & 0xff;
11290	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11292	break;
11293	}
11294
11295	case OP_INTO:
11296	{
11297	if (pMixedCtx->eflags.Bits.u1OF)
11298	{
11299	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11301	}
11302	break;
11303	}
11304
11305	default:
11306	{
11307	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11308	EMCODETYPE_SUPERVISOR);
11309	rc = VBOXSTRICTRC_VAL(rc2);
11310	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11311	Log4(("#GP rc=%Rrc\n", rc));
11312	break;
11313	}
11314	}
11315	}
11316	else
11317	rc = VERR_EM_INTERPRETER;
11318
11319	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11320	("#GP Unexpected rc=%Rrc\n", rc));
11321	return rc;
11322	}
11323
11324
11325	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11326	/**
11327	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11328	* the exception reported in the VMX transient structure back into the VM.
11329	*
11330	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11331	* up-to-date.
11332	*/
11333	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11334	{
11335	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11336
11337	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11338	hmR0VmxCheckExitDueToEventDelivery(). */
11339	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11340	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11341	AssertRCReturn(rc, rc);
11342	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11343
11344	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11345	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11346	return VINF_SUCCESS;
11347	}
11348	#endif
11349
11350
11351	/**
11352	* VM-exit exception handler for #PF (Page-fault exception).
11353	*/
11354	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11355	{
11356	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11357	PVM pVM = pVCpu->CTX_SUFF(pVM);
11358	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11359	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11360	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11361	AssertRCReturn(rc, rc);
11362
11363	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11364	if (pVM->hm.s.fNestedPaging)
11365	{
11366	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11367	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11368	{
11369	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11370	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11371	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11372	}
11373	else
11374	{
11375	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11376	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11377	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11378	}
11379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11380	return rc;
11381	}
11382	#else
11383	Assert(!pVM->hm.s.fNestedPaging);
11384	NOREF(pVM);
11385	#endif
11386
11387	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11388	AssertRCReturn(rc, rc);
11389
11390	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11391	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11392
11393	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11394	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11395	(RTGCPTR)pVmxTransient->uExitQualification);
11396
11397	Log4(("#PF: rc=%Rrc\n", rc));
11398	if (rc == VINF_SUCCESS)
11399	{
11400	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11401	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11402	* memory? We don't update the whole state here... */
11403	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11404	\| HM_CHANGED_GUEST_RSP
11405	\| HM_CHANGED_GUEST_RFLAGS
11406	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11407	TRPMResetTrap(pVCpu);
11408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11409	return rc;
11410	}
11411	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11412	{
11413	if (!pVmxTransient->fVectoringPF)
11414	{
11415	/* It's a guest page fault and needs to be reflected to the guest. */
11416	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11417	TRPMResetTrap(pVCpu);
11418	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11419	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11420	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11421	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11422	}
11423	else
11424	{
11425	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11426	TRPMResetTrap(pVCpu);
11427	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11428	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11429	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11430	}
11431
11432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11433	return VINF_SUCCESS;
11434	}
11435
11436	TRPMResetTrap(pVCpu);
11437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11438	return rc;
11439	}
11440
11441	/** @} */
11442

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 50271

Download in other formats: