HMVMXR0.cpp@ 49889

Last change on this file since 49889 was 49889, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Comments.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 471.2 KB

Line
1	/* $Id: HMVMXR0.cpp 49889 2013-12-12 17:12:08Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if (pVM->hm.s.fAllow64BitGuests)
2336	{
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	}
2342	#endif
2343	}
2344
2345	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2346	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2347	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2348
2349	if ((val & zap) != val)
2350	{
2351	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2352	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set processor-based VM-execution controls. */
2361	pVCpu->hm.s.vmx.u32ProcCtls = val;
2362
2363	/*
2364	* Secondary processor-based VM-execution controls.
2365	*/
2366	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2367	{
2368	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2373
2374	if (pVM->hm.s.fNestedPaging)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2376	else
2377	{
2378	/*
2379	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2380	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2381	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2382	*/
2383	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2385	}
2386
2387	if (pVM->hm.s.vmx.fVpid)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2389
2390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2392
2393	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2394	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2395	* done dynamically. */
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2397	{
2398	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2399	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2401	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2402	AssertRCReturn(rc, rc);
2403	}
2404
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2411	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2412	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2413	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2414	}
2415
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2417	AssertRCReturn(rc, rc);
2418
2419	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2420	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2421	}
2422	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2423	{
2424	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2425	"available\n"));
2426	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	return VINF_SUCCESS;
2431	}
2432
2433
2434	/**
2435	* Sets up miscellaneous (everything other than Pin & Processor-based
2436	* VM-execution) control fields in the VMCS.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	*/
2442	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2443	{
2444	NOREF(pVM);
2445	AssertPtr(pVM);
2446	AssertPtr(pVCpu);
2447
2448	int rc = VERR_GENERAL_FAILURE;
2449
2450	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2451	#if 0
2452	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2455
2456	/*
2457	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2458	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2459	* We thus use the exception bitmap to control it rather than use both.
2460	*/
2461	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2463
2464	/** @todo Explore possibility of using IO-bitmaps. */
2465	/* All IO & IOIO instructions cause VM-exits. */
2466	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/* Initialize the MSR-bitmap area. */
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2473	#endif
2474
2475	/* Setup MSR auto-load/store area. */
2476	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2477	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2479	AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2481	AssertRCReturn(rc, rc);
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2486	AssertRCReturn(rc, rc);
2487
2488	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2489	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2490	AssertRCReturn(rc, rc);
2491
2492	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2493	#if 0
2494	/* Setup debug controls */
2495	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2498	AssertRCReturn(rc, rc);
2499	#endif
2500
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Sets up the initial exception bitmap in the VMCS based on static conditions
2507	* (i.e. conditions that cannot ever change after starting the VM).
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	*/
2513	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2514	{
2515	AssertPtr(pVM);
2516	AssertPtr(pVCpu);
2517
2518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2519
2520	uint32_t u32XcptBitmap = 0;
2521
2522	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2523	if (!pVM->hm.s.fNestedPaging)
2524	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2525
2526	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2527	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2528	AssertRCReturn(rc, rc);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Sets up the initial guest-state mask. The guest-state mask is consulted
2535	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2536	* for the nested virtualization case (as it would cause a VM-exit).
2537	*
2538	* @param pVCpu Pointer to the VMCPU.
2539	*/
2540	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2541	{
2542	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2543	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2544	return VINF_SUCCESS;
2545	}
2546
2547
2548	/**
2549	* Does per-VM VT-x initialization.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	*/
2554	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2555	{
2556	LogFlowFunc(("pVM=%p\n", pVM));
2557
2558	int rc = hmR0VmxStructsAlloc(pVM);
2559	if (RT_FAILURE(rc))
2560	{
2561	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2562	return rc;
2563	}
2564
2565	return VINF_SUCCESS;
2566	}
2567
2568
2569	/**
2570	* Does per-VM VT-x termination.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM Pointer to the VM.
2574	*/
2575	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2576	{
2577	LogFlowFunc(("pVM=%p\n", pVM));
2578
2579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2580	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2581	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2582	#endif
2583	hmR0VmxStructsFree(pVM);
2584	return VINF_SUCCESS;
2585	}
2586
2587
2588	/**
2589	* Sets up the VM for execution under VT-x.
2590	* This function is only called once per-VM during initialization.
2591	*
2592	* @returns VBox status code.
2593	* @param pVM Pointer to the VM.
2594	*/
2595	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2596	{
2597	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2599
2600	LogFlowFunc(("pVM=%p\n", pVM));
2601
2602	/*
2603	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2604	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2605	*/
2606	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2607	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2608	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2609	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2610	{
2611	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2612	return VERR_INTERNAL_ERROR;
2613	}
2614
2615	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2616	/*
2617	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2618	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2619	*/
2620	if ( pVM->hm.s.fAllow64BitGuests
2621	&& !HMVMX_IS_64BIT_HOST_MODE())
2622	{
2623	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2624	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2625	}
2626	#endif
2627
2628	/* Initialize these always, see hmR3InitFinalizeR0().*/
2629	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2630	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2631
2632	/* Setup the tagged-TLB flush handlers. */
2633	int rc = hmR0VmxSetupTaggedTlb(pVM);
2634	if (RT_FAILURE(rc))
2635	{
2636	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2637	return rc;
2638	}
2639
2640	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2641	{
2642	PVMCPU pVCpu = &pVM->aCpus[i];
2643	AssertPtr(pVCpu);
2644	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2645
2646	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2647	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2648
2649	/* Set revision dword at the beginning of the VMCS structure. */
2650	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2651
2652	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2653	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2654	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2655	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2656
2657	/* Load this VMCS as the current VMCS. */
2658	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2659	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2660	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2661
2662	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2671	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2672	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2673
2674	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2675	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2676	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2677
2678	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2679	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2680	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2681
2682	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2683	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686	#endif
2687
2688	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2689	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2690	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2694
2695	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2696	}
2697
2698	return VINF_SUCCESS;
2699	}
2700
2701
2702	/**
2703	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2704	* the VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM Pointer to the VM.
2708	* @param pVCpu Pointer to the VMCPU.
2709	*/
2710	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2711	{
2712	NOREF(pVM); NOREF(pVCpu);
2713
2714	RTCCUINTREG uReg = ASMGetCR0();
2715	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2716	AssertRCReturn(rc, rc);
2717
2718	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2720	if (HMVMX_IS_64BIT_HOST_MODE())
2721	{
2722	uint64_t uRegCR3 = HMR0Get64bitCR3();
2723	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2724	}
2725	else
2726	#endif
2727	{
2728	uReg = ASMGetCR3();
2729	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2730	}
2731	AssertRCReturn(rc, rc);
2732
2733	uReg = ASMGetCR4();
2734	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2735	AssertRCReturn(rc, rc);
2736	return rc;
2737	}
2738
2739
2740	#if HC_ARCH_BITS == 64
2741	/**
2742	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2743	* requirements. See hmR0VmxSaveHostSegmentRegs().
2744	*/
2745	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2746	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2747	{ \
2748	bool fValidSelector = true; \
2749	if ((selValue) & X86_SEL_LDT) \
2750	{ \
2751	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2752	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2753	} \
2754	if (fValidSelector) \
2755	{ \
2756	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2757	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2758	} \
2759	(selValue) = 0; \
2760	}
2761	#endif
2762
2763
2764	/**
2765	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2766	* the host-state area in the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVM Pointer to the VM.
2770	* @param pVCpu Pointer to the VMCPU.
2771	*/
2772	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2773	{
2774	NOREF(pVM);
2775	int rc = VERR_INTERNAL_ERROR_5;
2776
2777	/*
2778	* Host DS, ES, FS and GS segment registers.
2779	*/
2780	#if HC_ARCH_BITS == 64
2781	RTSEL uSelDS = ASMGetDS();
2782	RTSEL uSelES = ASMGetES();
2783	RTSEL uSelFS = ASMGetFS();
2784	RTSEL uSelGS = ASMGetGS();
2785	#else
2786	RTSEL uSelDS = 0;
2787	RTSEL uSelES = 0;
2788	RTSEL uSelFS = 0;
2789	RTSEL uSelGS = 0;
2790	#endif
2791
2792	/* Recalculate which host-state bits need to be manually restored. */
2793	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2794
2795	/*
2796	* Host CS and SS segment registers.
2797	*/
2798	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2799	RTSEL uSelCS;
2800	RTSEL uSelSS;
2801	if (HMVMX_IS_64BIT_HOST_MODE())
2802	{
2803	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2804	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2805	}
2806	else
2807	{
2808	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2809	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2810	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2811	}
2812	#else
2813	RTSEL uSelCS = ASMGetCS();
2814	RTSEL uSelSS = ASMGetSS();
2815	#endif
2816
2817	/*
2818	* Host TR segment register.
2819	*/
2820	RTSEL uSelTR = ASMGetTR();
2821
2822	#if HC_ARCH_BITS == 64
2823	/*
2824	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2825	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2826	*/
2827	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2828	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2829	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2830	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2831	# undef VMXLOCAL_ADJUST_HOST_SEG
2832	#endif
2833
2834	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2835	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2836	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2837	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2838	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2839	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2840	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2841	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2842	Assert(uSelCS);
2843	Assert(uSelTR);
2844
2845	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2846	#if 0
2847	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2848	Assert(uSelSS != 0);
2849	#endif
2850
2851	/* Write these host selector fields into the host-state area in the VMCS. */
2852	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2853	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2854	#if HC_ARCH_BITS == 64
2855	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2857	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2859	#endif
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2861
2862	/*
2863	* Host GDTR and IDTR.
2864	*/
2865	RTGDTR Gdtr;
2866	RT_ZERO(Gdtr);
2867	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2868	if (HMVMX_IS_64BIT_HOST_MODE())
2869	{
2870	X86XDTR64 Gdtr64;
2871	X86XDTR64 Idtr64;
2872	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2873	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2874	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2875
2876	Gdtr.cbGdt = Gdtr64.cb;
2877	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2878	}
2879	else
2880	#endif
2881	{
2882	RTIDTR Idtr;
2883	ASMGetGDTR(&Gdtr);
2884	ASMGetIDTR(&Idtr);
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2887
2888	#if HC_ARCH_BITS == 64
2889	/*
2890	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2891	* maximum limit (0xffff) on every VM-exit.
2892	*/
2893	if (Gdtr.cbGdt != 0xffff)
2894	{
2895	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2896	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2897	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2898	}
2899
2900	/*
2901	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2902	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2903	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2904	*/
2905	if (Idtr.cbIdt < 0x0fff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2908	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2910	}
2911	#endif
2912	}
2913
2914	/*
2915	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2916	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2917	*/
2918	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2919	{
2920	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2921	return VERR_VMX_INVALID_HOST_STATE;
2922	}
2923
2924	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2925	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2926	if (HMVMX_IS_64BIT_HOST_MODE())
2927	{
2928	/* We need the 64-bit TR base for hybrid darwin. */
2929	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2930	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2931	}
2932	else
2933	#endif
2934	{
2935	uintptr_t uTRBase;
2936	#if HC_ARCH_BITS == 64
2937	uTRBase = X86DESC64_BASE(pDesc);
2938
2939	/*
2940	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2941	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2942	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2943	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2944	*
2945	* [1] See Intel spec. 3.5 "System Descriptor Types".
2946	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2947	*/
2948	Assert(pDesc->System.u4Type == 11);
2949	if ( pDesc->System.u16LimitLow != 0x67
2950	\|\| pDesc->System.u4LimitHigh)
2951	{
2952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2953	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2954
2955	/* Store the GDTR here as we need it while restoring TR. */
2956	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2957	}
2958	#else
2959	uTRBase = X86DESC_BASE(pDesc);
2960	#endif
2961	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2962	}
2963	AssertRCReturn(rc, rc);
2964
2965	/*
2966	* Host FS base and GS base.
2967	*/
2968	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2969	if (HMVMX_IS_64BIT_HOST_MODE())
2970	{
2971	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2972	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2973	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2974	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2975
2976	# if HC_ARCH_BITS == 64
2977	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2978	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2979	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2980	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2981	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2982	# endif
2983	}
2984	#endif
2985	return rc;
2986	}
2987
2988
2989	/**
2990	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2991	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2992	* the host after every successful VM-exit.
2993	*
2994	* @returns VBox status code.
2995	* @param pVM Pointer to the VM.
2996	* @param pVCpu Pointer to the VMCPU.
2997	*
2998	* @remarks No-long-jump zone!!!
2999	*/
3000	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3001	{
3002	NOREF(pVM);
3003
3004	AssertPtr(pVCpu);
3005	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3006
3007	int rc = VINF_SUCCESS;
3008	#if HC_ARCH_BITS == 64
3009	if (pVM->hm.s.fAllow64BitGuests)
3010	hmR0VmxLazySaveHostMsrs(pVCpu);
3011	#endif
3012
3013	if (pVCpu->hm.s.vmx.cMsrs > 0)
3014	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3015
3016	/*
3017	* Host Sysenter MSRs.
3018	*/
3019	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3020	AssertRCReturn(rc, rc);
3021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3022	if (HMVMX_IS_64BIT_HOST_MODE())
3023	{
3024	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3025	AssertRCReturn(rc, rc);
3026	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3027	}
3028	else
3029	{
3030	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3031	AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3033	}
3034	#elif HC_ARCH_BITS == 32
3035	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3036	AssertRCReturn(rc, rc);
3037	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3038	#else
3039	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3042	#endif
3043	AssertRCReturn(rc, rc);
3044
3045	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3046	* hmR0VmxSetupExitCtls() !! */
3047	return rc;
3048	}
3049
3050
3051	/**
3052	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3053	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3054	* controls".
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu Pointer to the VMCPU.
3058	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3059	* out-of-sync. Make sure to update the required fields
3060	* before using them.
3061	*
3062	* @remarks No-long-jump zone!!!
3063	*/
3064	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3065	{
3066	int rc = VINF_SUCCESS;
3067	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3068	{
3069	PVM pVM = pVCpu->CTX_SUFF(pVM);
3070	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3071	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3072
3073	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3074	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3075
3076	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3077	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3078	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3079	else
3080	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3081
3082	/*
3083	* The following should -not- be set (since we're not in SMM mode):
3084	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3085	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3086	*/
3087
3088	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3089	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3090	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3091
3092	if ((val & zap) != val)
3093	{
3094	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3095	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3096	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3097	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3098	}
3099
3100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3101	AssertRCReturn(rc, rc);
3102
3103	/* Update VCPU with the currently set VM-exit controls. */
3104	pVCpu->hm.s.vmx.u32EntryCtls = val;
3105	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3106	}
3107	return rc;
3108	}
3109
3110
3111	/**
3112	* Sets up the VM-exit controls in the VMCS.
3113	*
3114	* @returns VBox status code.
3115	* @param pVM Pointer to the VM.
3116	* @param pVCpu Pointer to the VMCPU.
3117	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3118	* out-of-sync. Make sure to update the required fields
3119	* before using them.
3120	*
3121	* @remarks requires EFER.
3122	*/
3123	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3124	{
3125	NOREF(pMixedCtx);
3126
3127	int rc = VINF_SUCCESS;
3128	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3129	{
3130	PVM pVM = pVCpu->CTX_SUFF(pVM);
3131	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3135	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3136
3137	/*
3138	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3139	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3140	*/
3141	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3142	if (HMVMX_IS_64BIT_HOST_MODE())
3143	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3144	else
3145	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3146	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3147	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3148	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3149	else
3150	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3151	#endif
3152
3153	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3154	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3155
3156	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3157	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3158	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3159	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3160	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3161
3162	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3163	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3164
3165	if ((val & zap) != val)
3166	{
3167	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3168	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3169	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3170	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3171	}
3172
3173	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3174	AssertRCReturn(rc, rc);
3175
3176	/* Update VCPU with the currently set VM-exit controls. */
3177	pVCpu->hm.s.vmx.u32ExitCtls = val;
3178	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3179	}
3180	return rc;
3181	}
3182
3183
3184	/**
3185	* Loads the guest APIC and related state.
3186	*
3187	* @returns VBox status code.
3188	* @param pVM Pointer to the VM.
3189	* @param pVCpu Pointer to the VMCPU.
3190	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3191	* out-of-sync. Make sure to update the required fields
3192	* before using them.
3193	*/
3194	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3195	{
3196	NOREF(pMixedCtx);
3197
3198	int rc = VINF_SUCCESS;
3199	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3200	{
3201	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3202	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3203	{
3204	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3205
3206	bool fPendingIntr = false;
3207	uint8_t u8Tpr = 0;
3208	uint8_t u8PendingIntr = 0;
3209	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3210	AssertRCReturn(rc, rc);
3211
3212	/*
3213	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3214	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3215	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3216	* the interrupt when we VM-exit for other reasons.
3217	*/
3218	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3219	uint32_t u32TprThreshold = 0;
3220	if (fPendingIntr)
3221	{
3222	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3223	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3224	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3225	if (u8PendingPriority <= u8TprPriority)
3226	u32TprThreshold = u8PendingPriority;
3227	else
3228	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3229	}
3230	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3231
3232	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3233	AssertRCReturn(rc, rc);
3234	}
3235
3236	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3237	}
3238	return rc;
3239	}
3240
3241
3242	/**
3243	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3244	*
3245	* @returns Guest's interruptibility-state.
3246	* @param pVCpu Pointer to the VMCPU.
3247	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3248	* out-of-sync. Make sure to update the required fields
3249	* before using them.
3250	*
3251	* @remarks No-long-jump zone!!!
3252	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3253	*/
3254	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3255	{
3256	/*
3257	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3258	* inhibit interrupts or clear any existing interrupt-inhibition.
3259	*/
3260	uint32_t uIntrState = 0;
3261	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3262	{
3263	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3264	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3265	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3266	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3267	{
3268	/*
3269	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3270	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3271	*/
3272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3273	}
3274	else if (pMixedCtx->eflags.Bits.u1IF)
3275	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3276	else
3277	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3278	}
3279	return uIntrState;
3280	}
3281
3282
3283	/**
3284	* Loads the guest's interruptibility-state into the guest-state area in the
3285	* VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param uIntrState The interruptibility-state to set.
3290	*/
3291	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3292	{
3293	NOREF(pVCpu);
3294	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3295	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3296	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3297	AssertRCReturn(rc, rc);
3298	return rc;
3299	}
3300
3301
3302	/**
3303	* Loads the guest's RIP into the guest-state area in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu Pointer to the VMCPU.
3307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308	* out-of-sync. Make sure to update the required fields
3309	* before using them.
3310	*
3311	* @remarks No-long-jump zone!!!
3312	*/
3313	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3314	{
3315	int rc = VINF_SUCCESS;
3316	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3317	{
3318	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3319	AssertRCReturn(rc, rc);
3320
3321	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3322	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3323	}
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Loads the guest's RSP into the guest-state area in the VMCS.
3330	*
3331	* @returns VBox status code.
3332	* @param pVCpu Pointer to the VMCPU.
3333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3334	* out-of-sync. Make sure to update the required fields
3335	* before using them.
3336	*
3337	* @remarks No-long-jump zone!!!
3338	*/
3339	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3340	{
3341	int rc = VINF_SUCCESS;
3342	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3343	{
3344	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3345	AssertRCReturn(rc, rc);
3346
3347	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3348	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3349	}
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu Pointer to the VMCPU.
3359	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360	* out-of-sync. Make sure to update the required fields
3361	* before using them.
3362	*
3363	* @remarks No-long-jump zone!!!
3364	*/
3365	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366	{
3367	int rc = VINF_SUCCESS;
3368	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3369	{
3370	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3371	Let us assert it as such and use 32-bit VMWRITE. */
3372	Assert(!(pMixedCtx->rflags.u64 >> 32));
3373	X86EFLAGS Eflags = pMixedCtx->eflags;
3374	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3375	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3376
3377	/*
3378	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3379	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3380	*/
3381	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3382	{
3383	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3384	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3385	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3386	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3387	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3388	}
3389
3390	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3391	AssertRCReturn(rc, rc);
3392
3393	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3394	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3395	}
3396	return rc;
3397	}
3398
3399
3400	/**
3401	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3402	*
3403	* @returns VBox status code.
3404	* @param pVCpu Pointer to the VMCPU.
3405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3406	* out-of-sync. Make sure to update the required fields
3407	* before using them.
3408	*
3409	* @remarks No-long-jump zone!!!
3410	*/
3411	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3412	{
3413	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3414	AssertRCReturn(rc, rc);
3415	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3416	AssertRCReturn(rc, rc);
3417	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3418	AssertRCReturn(rc, rc);
3419	return rc;
3420	}
3421
3422
3423	/**
3424	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3425	* CR0 is partially shared with the host and we have to consider the FPU bits.
3426	*
3427	* @returns VBox status code.
3428	* @param pVM Pointer to the VM.
3429	* @param pVCpu Pointer to the VMCPU.
3430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3431	* out-of-sync. Make sure to update the required fields
3432	* before using them.
3433	*
3434	* @remarks No-long-jump zone!!!
3435	*/
3436	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437	{
3438	/*
3439	* Guest CR0.
3440	* Guest FPU.
3441	*/
3442	int rc = VINF_SUCCESS;
3443	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3444	{
3445	Assert(!(pMixedCtx->cr0 >> 32));
3446	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3447	PVM pVM = pVCpu->CTX_SUFF(pVM);
3448
3449	/* The guest's view (read access) of its CR0 is unblemished. */
3450	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3451	AssertRCReturn(rc, rc);
3452	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3453
3454	/* Setup VT-x's view of the guest CR0. */
3455	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3456	if (pVM->hm.s.fNestedPaging)
3457	{
3458	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3459	{
3460	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3461	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3462	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3463	}
3464	else
3465	{
3466	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3467	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3468	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3469	}
3470
3471	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3472	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3473	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3474
3475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3476	AssertRCReturn(rc, rc);
3477	}
3478	else
3479	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3480
3481	/*
3482	* Guest FPU bits.
3483	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3484	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3485	*/
3486	u32GuestCR0 \|= X86_CR0_NE;
3487	bool fInterceptNM = false;
3488	if (CPUMIsGuestFPUStateActive(pVCpu))
3489	{
3490	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3491	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3492	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3493	}
3494	else
3495	{
3496	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3497	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3498	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3499	}
3500
3501	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3502	bool fInterceptMF = false;
3503	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3504	fInterceptMF = true;
3505
3506	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3507	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3508	{
3509	Assert(PDMVmmDevHeapIsEnabled(pVM));
3510	Assert(pVM->hm.s.vmx.pRealModeTSS);
3511	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3512	fInterceptNM = true;
3513	fInterceptMF = true;
3514	}
3515	else
3516	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3517
3518	if (fInterceptNM)
3519	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3520	else
3521	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3522
3523	if (fInterceptMF)
3524	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3525	else
3526	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3527
3528	/* Additional intercepts for debugging, define these yourself explicitly. */
3529	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3531	\| RT_BIT(X86_XCPT_BP)
3532	\| RT_BIT(X86_XCPT_DB)
3533	\| RT_BIT(X86_XCPT_DE)
3534	\| RT_BIT(X86_XCPT_NM)
3535	\| RT_BIT(X86_XCPT_UD)
3536	\| RT_BIT(X86_XCPT_NP)
3537	\| RT_BIT(X86_XCPT_SS)
3538	\| RT_BIT(X86_XCPT_GP)
3539	\| RT_BIT(X86_XCPT_PF)
3540	\| RT_BIT(X86_XCPT_MF)
3541	;
3542	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3543	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3544	#endif
3545
3546	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3547
3548	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3549	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3550	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3551	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3552	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3553	else
3554	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3555
3556	u32GuestCR0 \|= uSetCR0;
3557	u32GuestCR0 &= uZapCR0;
3558	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3559
3560	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3561	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3562	AssertRCReturn(rc, rc);
3563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3564	AssertRCReturn(rc, rc);
3565	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3566
3567	/*
3568	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3569	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3570	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3571	*/
3572	uint32_t u32CR0Mask = 0;
3573	u32CR0Mask = X86_CR0_PE
3574	\| X86_CR0_NE
3575	\| X86_CR0_WP
3576	\| X86_CR0_PG
3577	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3578	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3579	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3580
3581	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3582	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3583	* and @bugref{6944}. */
3584	#if 0
3585	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3586	u32CR0Mask &= ~X86_CR0_PE;
3587	#endif
3588	if (pVM->hm.s.fNestedPaging)
3589	u32CR0Mask &= ~X86_CR0_WP;
3590
3591	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3592	if (fInterceptNM)
3593	{
3594	u32CR0Mask \|= X86_CR0_TS
3595	\| X86_CR0_MP;
3596	}
3597
3598	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3599	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3600	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3601	AssertRCReturn(rc, rc);
3602	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3603
3604	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3605	}
3606	return rc;
3607	}
3608
3609
3610	/**
3611	* Loads the guest control registers (CR3, CR4) into the guest-state area
3612	* in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVM Pointer to the VM.
3616	* @param pVCpu Pointer to the VMCPU.
3617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618	* out-of-sync. Make sure to update the required fields
3619	* before using them.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624	{
3625	int rc = VINF_SUCCESS;
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627
3628	/*
3629	* Guest CR2.
3630	* It's always loaded in the assembler code. Nothing to do here.
3631	*/
3632
3633	/*
3634	* Guest CR3.
3635	*/
3636	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3637	{
3638	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3639	if (pVM->hm.s.fNestedPaging)
3640	{
3641	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3642
3643	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3644	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3645	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3646	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3647
3648	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3649	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3650	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3651
3652	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3653	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3654	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3655	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3656
3657	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3658	AssertRCReturn(rc, rc);
3659	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3660
3661	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3662	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3663	{
3664	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3665	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3666	{
3667	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3668	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3669	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3670	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3671	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3672	}
3673
3674	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3675	have Unrestricted Execution to handle the guest when it's not using paging. */
3676	GCPhysGuestCR3 = pMixedCtx->cr3;
3677	}
3678	else
3679	{
3680	/*
3681	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3682	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3683	* EPT takes care of translating it to host-physical addresses.
3684	*/
3685	RTGCPHYS GCPhys;
3686	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3687	Assert(PDMVmmDevHeapIsEnabled(pVM));
3688
3689	/* We obtain it here every time as the guest could have relocated this PCI region. */
3690	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3691	AssertRCReturn(rc, rc);
3692
3693	GCPhysGuestCR3 = GCPhys;
3694	}
3695
3696	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3697	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3698	}
3699	else
3700	{
3701	/* Non-nested paging case, just use the hypervisor's CR3. */
3702	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3703
3704	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3705	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3706	}
3707	AssertRCReturn(rc, rc);
3708
3709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3710	}
3711
3712	/*
3713	* Guest CR4.
3714	*/
3715	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3716	{
3717	Assert(!(pMixedCtx->cr4 >> 32));
3718	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3719
3720	/* The guest's view of its CR4 is unblemished. */
3721	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3722	AssertRCReturn(rc, rc);
3723	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3724
3725	/* Setup VT-x's view of the guest CR4. */
3726	/*
3727	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3728	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3729	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3730	*/
3731	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3732	{
3733	Assert(pVM->hm.s.vmx.pRealModeTSS);
3734	Assert(PDMVmmDevHeapIsEnabled(pVM));
3735	u32GuestCR4 &= ~X86_CR4_VME;
3736	}
3737
3738	if (pVM->hm.s.fNestedPaging)
3739	{
3740	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3741	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3742	{
3743	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3744	u32GuestCR4 \|= X86_CR4_PSE;
3745	/* Our identity mapping is a 32-bit page directory. */
3746	u32GuestCR4 &= ~X86_CR4_PAE;
3747	}
3748	/* else use guest CR4.*/
3749	}
3750	else
3751	{
3752	/*
3753	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3754	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3755	*/
3756	switch (pVCpu->hm.s.enmShadowMode)
3757	{
3758	case PGMMODE_REAL: /* Real-mode. */
3759	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3760	case PGMMODE_32_BIT: /* 32-bit paging. */
3761	{
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	break;
3764	}
3765
3766	case PGMMODE_PAE: /* PAE paging. */
3767	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3768	{
3769	u32GuestCR4 \|= X86_CR4_PAE;
3770	break;
3771	}
3772
3773	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3774	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3775	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3776	break;
3777	#endif
3778	default:
3779	AssertFailed();
3780	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3781	}
3782	}
3783
3784	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3785	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3786	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3787	u32GuestCR4 \|= uSetCR4;
3788	u32GuestCR4 &= uZapCR4;
3789
3790	/* Write VT-x's view of the guest CR4 into the VMCS. */
3791	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3792	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3793	AssertRCReturn(rc, rc);
3794
3795	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3796	uint32_t u32CR4Mask = 0;
3797	u32CR4Mask = X86_CR4_VME
3798	\| X86_CR4_PAE
3799	\| X86_CR4_PGE
3800	\| X86_CR4_PSE
3801	\| X86_CR4_VMXE;
3802	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3803	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3804	AssertRCReturn(rc, rc);
3805
3806	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3807	}
3808	return rc;
3809	}
3810
3811
3812	/**
3813	* Loads the guest debug registers into the guest-state area in the VMCS.
3814	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3815	*
3816	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3817	*
3818	* @returns VBox status code.
3819	* @param pVCpu Pointer to the VMCPU.
3820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3821	* out-of-sync. Make sure to update the required fields
3822	* before using them.
3823	*
3824	* @remarks No-long-jump zone!!!
3825	*/
3826	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3827	{
3828	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3829	return VINF_SUCCESS;
3830
3831	#ifdef VBOX_STRICT
3832	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3833	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3834	{
3835	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3836	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3837	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3838	}
3839	#endif
3840
3841	int rc;
3842	PVM pVM = pVCpu->CTX_SUFF(pVM);
3843	bool fInterceptDB = false;
3844	bool fInterceptMovDRx = false;
3845	if ( pVCpu->hm.s.fSingleInstruction
3846	\|\| DBGFIsStepping(pVCpu))
3847	{
3848	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3849	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3850	{
3851	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3852	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3853	AssertRCReturn(rc, rc);
3854	Assert(fInterceptDB == false);
3855	}
3856	else
3857	{
3858	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3859	pVCpu->hm.s.fClearTrapFlag = true;
3860	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3861	fInterceptDB = true;
3862	}
3863	}
3864
3865	if ( fInterceptDB
3866	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3867	{
3868	/*
3869	* Use the combined guest and host DRx values found in the hypervisor
3870	* register set because the debugger has breakpoints active or someone
3871	* is single stepping on the host side without a monitor trap flag.
3872	*
3873	* Note! DBGF expects a clean DR6 state before executing guest code.
3874	*/
3875	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3876	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3877	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3878	{
3879	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3880	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3881	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3882	}
3883	else
3884	#endif
3885	if (!CPUMIsHyperDebugStateActive(pVCpu))
3886	{
3887	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3888	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3889	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3890	}
3891
3892	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3893	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3894	AssertRCReturn(rc, rc);
3895
3896	pVCpu->hm.s.fUsingHyperDR7 = true;
3897	fInterceptDB = true;
3898	fInterceptMovDRx = true;
3899	}
3900	else
3901	{
3902	/*
3903	* If the guest has enabled debug registers, we need to load them prior to
3904	* executing guest code so they'll trigger at the right time.
3905	*/
3906	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3907	{
3908	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3909	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3910	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3911	{
3912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3913	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3914	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3915	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3916	}
3917	else
3918	#endif
3919	if (!CPUMIsGuestDebugStateActive(pVCpu))
3920	{
3921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3922	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3924	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3925	}
3926	Assert(!fInterceptDB);
3927	Assert(!fInterceptMovDRx);
3928	}
3929	/*
3930	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3931	* must intercept #DB in order to maintain a correct DR6 guest value.
3932	*/
3933	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3935	&& !CPUMIsGuestDebugStateActive(pVCpu))
3936	#else
3937	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3938	#endif
3939	{
3940	fInterceptMovDRx = true;
3941	fInterceptDB = true;
3942	}
3943
3944	/* Update guest DR7. */
3945	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3946	AssertRCReturn(rc, rc);
3947
3948	pVCpu->hm.s.fUsingHyperDR7 = false;
3949	}
3950
3951	/*
3952	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3953	*/
3954	if (fInterceptDB)
3955	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3956	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3957	{
3958	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3959	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3960	#endif
3961	}
3962	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3963	AssertRCReturn(rc, rc);
3964
3965	/*
3966	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3967	*/
3968	if (fInterceptMovDRx)
3969	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3970	else
3971	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3973	AssertRCReturn(rc, rc);
3974
3975	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3976	return VINF_SUCCESS;
3977	}
3978
3979
3980	#ifdef VBOX_STRICT
3981	/**
3982	* Strict function to validate segment registers.
3983	*
3984	* @remarks ASSUMES CR0 is up to date.
3985	*/
3986	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3987	{
3988	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3989	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3990	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3991	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3992	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3993	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3994	{
3995	/* Protected mode checks */
3996	/* CS */
3997	Assert(pCtx->cs.Attr.n.u1Present);
3998	Assert(!(pCtx->cs.Attr.u & 0xf00));
3999	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4000	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4001	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4002	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4003	\|\| (pCtx->cs.Attr.n.u1Granularity));
4004	/* CS cannot be loaded with NULL in protected mode. */
4005	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4006	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4007	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4008	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4009	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4010	else
4011	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4012	/* SS */
4013	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4014	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4015	if ( !(pCtx->cr0 & X86_CR0_PE)
4016	\|\| pCtx->cs.Attr.n.u4Type == 3)
4017	{
4018	Assert(!pCtx->ss.Attr.n.u2Dpl);
4019	}
4020	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4021	{
4022	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4023	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4024	Assert(pCtx->ss.Attr.n.u1Present);
4025	Assert(!(pCtx->ss.Attr.u & 0xf00));
4026	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4027	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4028	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4029	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4030	\|\| (pCtx->ss.Attr.n.u1Granularity));
4031	}
4032	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4033	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4034	{
4035	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4036	Assert(pCtx->ds.Attr.n.u1Present);
4037	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4038	Assert(!(pCtx->ds.Attr.u & 0xf00));
4039	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4040	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4041	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4042	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4043	\|\| (pCtx->ds.Attr.n.u1Granularity));
4044	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4045	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4046	}
4047	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4048	{
4049	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4050	Assert(pCtx->es.Attr.n.u1Present);
4051	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4052	Assert(!(pCtx->es.Attr.u & 0xf00));
4053	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4054	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4055	\|\| !(pCtx->es.Attr.n.u1Granularity));
4056	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4057	\|\| (pCtx->es.Attr.n.u1Granularity));
4058	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4059	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4060	}
4061	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4062	{
4063	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4064	Assert(pCtx->fs.Attr.n.u1Present);
4065	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4066	Assert(!(pCtx->fs.Attr.u & 0xf00));
4067	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4068	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4069	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4070	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4071	\|\| (pCtx->fs.Attr.n.u1Granularity));
4072	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4073	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4074	}
4075	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4076	{
4077	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4078	Assert(pCtx->gs.Attr.n.u1Present);
4079	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4080	Assert(!(pCtx->gs.Attr.u & 0xf00));
4081	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4082	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4083	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4084	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4085	\|\| (pCtx->gs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4087	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4088	}
4089	/* 64-bit capable CPUs. */
4090	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4091	if (HMVMX_IS_64BIT_HOST_MODE())
4092	{
4093	Assert(!(pCtx->cs.u64Base >> 32));
4094	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4095	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4096	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4097	}
4098	# endif
4099	}
4100	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4101	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4102	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4103	{
4104	/* Real and v86 mode checks. */
4105	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4106	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4108	{
4109	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4110	}
4111	else
4112	{
4113	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4114	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4115	}
4116
4117	/* CS */
4118	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4119	Assert(pCtx->cs.u32Limit == 0xffff);
4120	Assert(u32CSAttr == 0xf3);
4121	/* SS */
4122	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4123	Assert(pCtx->ss.u32Limit == 0xffff);
4124	Assert(u32SSAttr == 0xf3);
4125	/* DS */
4126	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4127	Assert(pCtx->ds.u32Limit == 0xffff);
4128	Assert(u32DSAttr == 0xf3);
4129	/* ES */
4130	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4131	Assert(pCtx->es.u32Limit == 0xffff);
4132	Assert(u32ESAttr == 0xf3);
4133	/* FS */
4134	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4135	Assert(pCtx->fs.u32Limit == 0xffff);
4136	Assert(u32FSAttr == 0xf3);
4137	/* GS */
4138	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4139	Assert(pCtx->gs.u32Limit == 0xffff);
4140	Assert(u32GSAttr == 0xf3);
4141	/* 64-bit capable CPUs. */
4142	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143	if (HMVMX_IS_64BIT_HOST_MODE())
4144	{
4145	Assert(!(pCtx->cs.u64Base >> 32));
4146	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4147	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4148	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4149	}
4150	# endif
4151	}
4152	}
4153	#endif /* VBOX_STRICT */
4154
4155
4156	/**
4157	* Writes a guest segment register into the guest-state area in the VMCS.
4158	*
4159	* @returns VBox status code.
4160	* @param pVCpu Pointer to the VMCPU.
4161	* @param idxSel Index of the selector in the VMCS.
4162	* @param idxLimit Index of the segment limit in the VMCS.
4163	* @param idxBase Index of the segment base in the VMCS.
4164	* @param idxAccess Index of the access rights of the segment in the VMCS.
4165	* @param pSelReg Pointer to the segment selector.
4166	*
4167	* @remarks No-long-jump zone!!!
4168	*/
4169	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4170	uint32_t idxAccess, PCPUMSELREG pSelReg)
4171	{
4172	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4175	AssertRCReturn(rc, rc);
4176	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4177	AssertRCReturn(rc, rc);
4178
4179	uint32_t u32Access = pSelReg->Attr.u;
4180	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181	{
4182	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4183	u32Access = 0xf3;
4184	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4185	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4186	}
4187	else
4188	{
4189	/*
4190	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4191	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4192	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4193	* loaded in protected-mode have their attribute as 0.
4194	*/
4195	if (!u32Access)
4196	u32Access = X86DESCATTR_UNUSABLE;
4197	}
4198
4199	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4200	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4201	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4202
4203	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4204	AssertRCReturn(rc, rc);
4205	return rc;
4206	}
4207
4208
4209	/**
4210	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4211	* into the guest-state area in the VMCS.
4212	*
4213	* @returns VBox status code.
4214	* @param pVM Pointer to the VM.
4215	* @param pVCPU Pointer to the VMCPU.
4216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217	* out-of-sync. Make sure to update the required fields
4218	* before using them.
4219	*
4220	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4224	{
4225	int rc = VERR_INTERNAL_ERROR_5;
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/*
4229	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4230	*/
4231	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4232	{
4233	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4234	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4235	{
4236	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4240	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4242	}
4243
4244	#ifdef VBOX_WITH_REM
4245	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4246	{
4247	Assert(pVM->hm.s.vmx.pRealModeTSS);
4248	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4249	if ( pVCpu->hm.s.vmx.fWasInRealMode
4250	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4251	{
4252	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4253	in real-mode (e.g. OpenBSD 4.0) */
4254	REMFlushTBs(pVM);
4255	Log4(("Load: Switch to protected mode detected!\n"));
4256	pVCpu->hm.s.vmx.fWasInRealMode = false;
4257	}
4258	}
4259	#endif
4260	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4261	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4262	AssertRCReturn(rc, rc);
4263	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4264	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4265	AssertRCReturn(rc, rc);
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4267	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4270	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4273	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4276	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4277	AssertRCReturn(rc, rc);
4278
4279	#ifdef VBOX_STRICT
4280	/* Validate. */
4281	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4282	#endif
4283
4284	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4285	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4286	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4287	}
4288
4289	/*
4290	* Guest TR.
4291	*/
4292	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4293	{
4294	/*
4295	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4296	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4297	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4298	*/
4299	uint16_t u16Sel = 0;
4300	uint32_t u32Limit = 0;
4301	uint64_t u64Base = 0;
4302	uint32_t u32AccessRights = 0;
4303
4304	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4305	{
4306	u16Sel = pMixedCtx->tr.Sel;
4307	u32Limit = pMixedCtx->tr.u32Limit;
4308	u64Base = pMixedCtx->tr.u64Base;
4309	u32AccessRights = pMixedCtx->tr.Attr.u;
4310	}
4311	else
4312	{
4313	Assert(pVM->hm.s.vmx.pRealModeTSS);
4314	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4315
4316	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4317	RTGCPHYS GCPhys;
4318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4319	AssertRCReturn(rc, rc);
4320
4321	X86DESCATTR DescAttr;
4322	DescAttr.u = 0;
4323	DescAttr.n.u1Present = 1;
4324	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4325
4326	u16Sel = 0;
4327	u32Limit = HM_VTX_TSS_SIZE;
4328	u64Base = GCPhys; /* in real-mode phys = virt. */
4329	u32AccessRights = DescAttr.u;
4330	}
4331
4332	/* Validate. */
4333	Assert(!(u16Sel & RT_BIT(2)));
4334	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4335	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4336	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4337	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4338	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4339	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4340	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4341	Assert( (u32Limit & 0xfff) == 0xfff
4342	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4343	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4344	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4345
4346	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4349	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4350
4351	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4352	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4353	}
4354
4355	/*
4356	* Guest GDTR.
4357	*/
4358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4359	{
4360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4361	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4362
4363	/* Validate. */
4364	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4365
4366	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4367	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4368	}
4369
4370	/*
4371	* Guest LDTR.
4372	*/
4373	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4374	{
4375	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4376	uint32_t u32Access = 0;
4377	if (!pMixedCtx->ldtr.Attr.u)
4378	u32Access = X86DESCATTR_UNUSABLE;
4379	else
4380	u32Access = pMixedCtx->ldtr.Attr.u;
4381
4382	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4384	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4385	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4386
4387	/* Validate. */
4388	if (!(u32Access & X86DESCATTR_UNUSABLE))
4389	{
4390	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4393	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4394	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4395	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4396	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4397	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4398	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4399	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4400	}
4401
4402	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4403	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4404	}
4405
4406	/*
4407	* Guest IDTR.
4408	*/
4409	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4410	{
4411	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4412	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4413
4414	/* Validate. */
4415	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4416
4417	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4419	}
4420
4421	return VINF_SUCCESS;
4422	}
4423
4424
4425	/**
4426	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4427	* areas. These MSRs will automatically be loaded to the host CPU on every
4428	* successful VM entry and stored from the host CPU on every successful VM-exit.
4429	*
4430	* This also creates/updates MSR slots for the host MSRs. The actual host
4431	* MSR values are -not- updated here for performance reasons. See
4432	* hmR0VmxSaveHostMsrs().
4433	*
4434	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4445	{
4446	AssertPtr(pVCpu);
4447	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4448
4449	/*
4450	* MSRs that we use the auto-load/store MSR area in the VMCS.
4451	*/
4452	PVM pVM = pVCpu->CTX_SUFF(pVM);
4453	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4454	{
4455	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4456	if (pVM->hm.s.fAllow64BitGuests)
4457	{
4458	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4459	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4462	# ifdef DEBUG
4463	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4464	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4465	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4466	# endif
4467	}
4468	#endif
4469	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4470	}
4471
4472	/*
4473	* Guest Sysenter MSRs.
4474	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4475	* VM-exits on WRMSRs for these MSRs.
4476	*/
4477	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4478	{
4479	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4480	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4481	}
4482
4483	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4484	{
4485	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4486	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4487	}
4488
4489	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4490	{
4491	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4492	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4493	}
4494
4495	return VINF_SUCCESS;
4496	}
4497
4498
4499	/**
4500	* Loads the guest activity state into the guest-state area in the VMCS.
4501	*
4502	* @returns VBox status code.
4503	* @param pVCpu Pointer to the VMCPU.
4504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4505	* out-of-sync. Make sure to update the required fields
4506	* before using them.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4511	{
4512	NOREF(pCtx);
4513	/** @todo See if we can make use of other states, e.g.
4514	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4515	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4516	{
4517	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4518	AssertRCReturn(rc, rc);
4519
4520	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4521	}
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Sets up the appropriate function to run guest code.
4528	*
4529	* @returns VBox status code.
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4532	* out-of-sync. Make sure to update the required fields
4533	* before using them.
4534	*
4535	* @remarks No-long-jump zone!!!
4536	*/
4537	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4538	{
4539	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4540	{
4541	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4542	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4543	#endif
4544	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4545	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4546	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4547	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4548	{
4549	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4550	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4551	}
4552	#else
4553	/* 64-bit host or hybrid host. */
4554	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4555	#endif
4556	}
4557	else
4558	{
4559	/* Guest is not in long mode, use the 32-bit handler. */
4560	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4561	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4562	{
4563	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4564	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4565	}
4566	#else
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4568	#endif
4569	}
4570	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4571	return VINF_SUCCESS;
4572	}
4573
4574
4575	/**
4576	* Wrapper for running the guest code in VT-x.
4577	*
4578	* @returns VBox strict status code.
4579	* @param pVM Pointer to the VM.
4580	* @param pVCpu Pointer to the VMCPU.
4581	* @param pCtx Pointer to the guest-CPU context.
4582	*
4583	* @remarks No-long-jump zone!!!
4584	*/
4585	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4586	{
4587	/*
4588	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4589	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4590	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4591	*/
4592	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4593	/** @todo Add stats for resume vs launch. */
4594	#ifdef VBOX_WITH_KERNEL_USING_XMM
4595	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4596	#else
4597	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4598	#endif
4599	}
4600
4601
4602	/**
4603	* Reports world-switch error and dumps some useful debug info.
4604	*
4605	* @param pVM Pointer to the VM.
4606	* @param pVCpu Pointer to the VMCPU.
4607	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4608	* @param pCtx Pointer to the guest-CPU context.
4609	* @param pVmxTransient Pointer to the VMX transient structure (only
4610	* exitReason updated).
4611	*/
4612	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4613	{
4614	Assert(pVM);
4615	Assert(pVCpu);
4616	Assert(pCtx);
4617	Assert(pVmxTransient);
4618	HMVMX_ASSERT_PREEMPT_SAFE();
4619
4620	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4621	switch (rcVMRun)
4622	{
4623	case VERR_VMX_INVALID_VMXON_PTR:
4624	AssertFailed();
4625	break;
4626	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4627	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4628	{
4629	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4630	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4631	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4632	AssertRC(rc);
4633
4634	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4635	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4636	Cannot do it here as we may have been long preempted. */
4637
4638	#ifdef VBOX_STRICT
4639	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4640	pVmxTransient->uExitReason));
4641	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4642	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4643	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4644	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4645	else
4646	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4647	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4648	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4649
4650	/* VMX control bits. */
4651	uint32_t u32Val;
4652	uint64_t u64Val;
4653	HMVMXHCUINTREG uHCReg;
4654	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4655	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4656	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4657	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4659	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4686	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4687	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4688	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4689	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4691	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4694	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4695	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4696
4697	/* Guest bits. */
4698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4699	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4700	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4701	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4702	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4703	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4704	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4705	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4706
4707	/* Host bits. */
4708	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4709	Log4(("Host CR0 %#RHr\n", uHCReg));
4710	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4711	Log4(("Host CR3 %#RHr\n", uHCReg));
4712	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4713	Log4(("Host CR4 %#RHr\n", uHCReg));
4714
4715	RTGDTR HostGdtr;
4716	PCX86DESCHC pDesc;
4717	ASMGetGDTR(&HostGdtr);
4718	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4719	Log4(("Host CS %#08x\n", u32Val));
4720	if (u32Val < HostGdtr.cbGdt)
4721	{
4722	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4723	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4724	}
4725
4726	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4727	Log4(("Host DS %#08x\n", u32Val));
4728	if (u32Val < HostGdtr.cbGdt)
4729	{
4730	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4731	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4732	}
4733
4734	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4735	Log4(("Host ES %#08x\n", u32Val));
4736	if (u32Val < HostGdtr.cbGdt)
4737	{
4738	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4739	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4740	}
4741
4742	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4743	Log4(("Host FS %#08x\n", u32Val));
4744	if (u32Val < HostGdtr.cbGdt)
4745	{
4746	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4747	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4748	}
4749
4750	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4751	Log4(("Host GS %#08x\n", u32Val));
4752	if (u32Val < HostGdtr.cbGdt)
4753	{
4754	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4755	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4756	}
4757
4758	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4759	Log4(("Host SS %#08x\n", u32Val));
4760	if (u32Val < HostGdtr.cbGdt)
4761	{
4762	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4763	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4764	}
4765
4766	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4767	Log4(("Host TR %#08x\n", u32Val));
4768	if (u32Val < HostGdtr.cbGdt)
4769	{
4770	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4771	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4772	}
4773
4774	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4775	Log4(("Host TR Base %#RHv\n", uHCReg));
4776	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4777	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4778	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4779	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4780	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4781	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4783	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4784	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4785	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4786	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4787	Log4(("Host RSP %#RHv\n", uHCReg));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4789	Log4(("Host RIP %#RHv\n", uHCReg));
4790	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4791	if (HMVMX_IS_64BIT_HOST_MODE())
4792	{
4793	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4794	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4795	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4796	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4797	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4798	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4799	}
4800	# endif
4801	#endif /* VBOX_STRICT */
4802	break;
4803	}
4804
4805	default:
4806	/* Impossible */
4807	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4808	break;
4809	}
4810	NOREF(pVM); NOREF(pCtx);
4811	}
4812
4813
4814	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4815	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4816	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4817	#endif
4818	#ifdef VBOX_STRICT
4819	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4820	{
4821	switch (idxField)
4822	{
4823	case VMX_VMCS_GUEST_RIP:
4824	case VMX_VMCS_GUEST_RSP:
4825	case VMX_VMCS_GUEST_SYSENTER_EIP:
4826	case VMX_VMCS_GUEST_SYSENTER_ESP:
4827	case VMX_VMCS_GUEST_GDTR_BASE:
4828	case VMX_VMCS_GUEST_IDTR_BASE:
4829	case VMX_VMCS_GUEST_CS_BASE:
4830	case VMX_VMCS_GUEST_DS_BASE:
4831	case VMX_VMCS_GUEST_ES_BASE:
4832	case VMX_VMCS_GUEST_FS_BASE:
4833	case VMX_VMCS_GUEST_GS_BASE:
4834	case VMX_VMCS_GUEST_SS_BASE:
4835	case VMX_VMCS_GUEST_LDTR_BASE:
4836	case VMX_VMCS_GUEST_TR_BASE:
4837	case VMX_VMCS_GUEST_CR3:
4838	return true;
4839	}
4840	return false;
4841	}
4842
4843	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4844	{
4845	switch (idxField)
4846	{
4847	/* Read-only fields. */
4848	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4849	return true;
4850	}
4851	/* Remaining readable fields should also be writable. */
4852	return hmR0VmxIsValidWriteField(idxField);
4853	}
4854	#endif /* VBOX_STRICT */
4855
4856
4857	/**
4858	* Executes the specified handler in 64-bit mode.
4859	*
4860	* @returns VBox status code.
4861	* @param pVM Pointer to the VM.
4862	* @param pVCpu Pointer to the VMCPU.
4863	* @param pCtx Pointer to the guest CPU context.
4864	* @param enmOp The operation to perform.
4865	* @param cbParam Number of parameters.
4866	* @param paParam Array of 32-bit parameters.
4867	*/
4868	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4869	uint32_t *paParam)
4870	{
4871	int rc, rc2;
4872	PHMGLOBALCPUINFO pCpu;
4873	RTHCPHYS HCPhysCpuPage;
4874	RTCCUINTREG uOldEflags;
4875
4876	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4877	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4878	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4879	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4880
4881	#ifdef VBOX_STRICT
4882	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4883	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4884
4885	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4886	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4887	#endif
4888
4889	/* Disable interrupts. */
4890	uOldEflags = ASMIntDisableFlags();
4891
4892	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4893	RTCPUID idHostCpu = RTMpCpuId();
4894	CPUMR0SetLApic(pVCpu, idHostCpu);
4895	#endif
4896
4897	pCpu = HMR0GetCurrentCpu();
4898	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4899
4900	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4901	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4902
4903	/* Leave VMX Root Mode. */
4904	VMXDisable();
4905
4906	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4907
4908	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4909	CPUMSetHyperEIP(pVCpu, enmOp);
4910	for (int i = (int)cbParam - 1; i >= 0; i--)
4911	CPUMPushHyper(pVCpu, paParam[i]);
4912
4913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4914
4915	/* Call the switcher. */
4916	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4917	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4918
4919	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4920	/* Make sure the VMX instructions don't cause #UD faults. */
4921	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4922
4923	/* Re-enter VMX Root Mode */
4924	rc2 = VMXEnable(HCPhysCpuPage);
4925	if (RT_FAILURE(rc2))
4926	{
4927	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4928	ASMSetFlags(uOldEflags);
4929	return rc2;
4930	}
4931
4932	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4933	AssertRC(rc2);
4934	Assert(!(ASMGetFlags() & X86_EFL_IF));
4935	ASMSetFlags(uOldEflags);
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4942	* supporting 64-bit guests.
4943	*
4944	* @returns VBox status code.
4945	* @param fResume Whether to VMLAUNCH or VMRESUME.
4946	* @param pCtx Pointer to the guest-CPU context.
4947	* @param pCache Pointer to the VMCS cache.
4948	* @param pVM Pointer to the VM.
4949	* @param pVCpu Pointer to the VMCPU.
4950	*/
4951	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4952	{
4953	uint32_t aParam[6];
4954	PHMGLOBALCPUINFO pCpu = NULL;
4955	RTHCPHYS HCPhysCpuPage = 0;
4956	int rc = VERR_INTERNAL_ERROR_5;
4957
4958	pCpu = HMR0GetCurrentCpu();
4959	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4960
4961	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4962	pCache->uPos = 1;
4963	pCache->interPD = PGMGetInterPaeCR3(pVM);
4964	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4965	#endif
4966
4967	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4968	pCache->TestIn.HCPhysCpuPage = 0;
4969	pCache->TestIn.HCPhysVmcs = 0;
4970	pCache->TestIn.pCache = 0;
4971	pCache->TestOut.HCPhysVmcs = 0;
4972	pCache->TestOut.pCache = 0;
4973	pCache->TestOut.pCtx = 0;
4974	pCache->TestOut.eflags = 0;
4975	#endif
4976
4977	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4978	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4979	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4980	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4981	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4982	aParam[5] = 0;
4983
4984	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4985	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4986	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4987	#endif
4988	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4989
4990	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4991	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4992	Assert(pCtx->dr[4] == 10);
4993	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4994	#endif
4995
4996	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4997	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4998	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4999	pVCpu->hm.s.vmx.HCPhysVmcs));
5000	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5001	pCache->TestOut.HCPhysVmcs));
5002	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5003	pCache->TestOut.pCache));
5004	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5005	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5006	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5007	pCache->TestOut.pCtx));
5008	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5009	#endif
5010	return rc;
5011	}
5012
5013
5014	/**
5015	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5016	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5017	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5018	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5019	*
5020	* @returns VBox status code.
5021	* @param pVM Pointer to the VM.
5022	* @param pVCpu Pointer to the VMCPU.
5023	*/
5024	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5025	{
5026	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5027	{ \
5028	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5029	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5030	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5031	++cReadFields; \
5032	}
5033
5034	AssertPtr(pVM);
5035	AssertPtr(pVCpu);
5036	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5037	uint32_t cReadFields = 0;
5038
5039	/*
5040	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5041	* and serve to indicate exceptions to the rules.
5042	*/
5043
5044	/* Guest-natural selector base fields. */
5045	#if 0
5046	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5047	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5048	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5049	#endif
5050	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5062	#if 0
5063	/* Unused natural width guest-state fields. */
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5066	#endif
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5069
5070	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5071	#if 0
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5081	#endif
5082
5083	/* Natural width guest-state fields. */
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5085	#if 0
5086	/* Currently unused field. */
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5088	#endif
5089
5090	if (pVM->hm.s.fNestedPaging)
5091	{
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5093	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5094	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5095	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5096	}
5097	else
5098	{
5099	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5100	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5101	}
5102
5103	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5104	return VINF_SUCCESS;
5105	}
5106
5107
5108	/**
5109	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5110	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5111	* darwin, running 64-bit guests).
5112	*
5113	* @returns VBox status code.
5114	* @param pVCpu Pointer to the VMCPU.
5115	* @param idxField The VMCS field encoding.
5116	* @param u64Val 16, 32 or 64-bit value.
5117	*/
5118	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5119	{
5120	int rc;
5121	switch (idxField)
5122	{
5123	/*
5124	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5125	*/
5126	/* 64-bit Control fields. */
5127	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5128	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5129	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5130	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5131	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5132	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5133	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5134	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5135	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5136	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5137	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5138	case VMX_VMCS64_CTRL_EPTP_FULL:
5139	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5140	/* 64-bit Guest-state fields. */
5141	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5142	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5143	case VMX_VMCS64_GUEST_PAT_FULL:
5144	case VMX_VMCS64_GUEST_EFER_FULL:
5145	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5146	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5147	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5148	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5150	/* 64-bit Host-state fields. */
5151	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5152	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5153	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5154	{
5155	rc = VMXWriteVmcs32(idxField, u64Val);
5156	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5157	break;
5158	}
5159
5160	/*
5161	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5162	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5163	*/
5164	/* Natural-width Guest-state fields. */
5165	case VMX_VMCS_GUEST_CR3:
5166	case VMX_VMCS_GUEST_ES_BASE:
5167	case VMX_VMCS_GUEST_CS_BASE:
5168	case VMX_VMCS_GUEST_SS_BASE:
5169	case VMX_VMCS_GUEST_DS_BASE:
5170	case VMX_VMCS_GUEST_FS_BASE:
5171	case VMX_VMCS_GUEST_GS_BASE:
5172	case VMX_VMCS_GUEST_LDTR_BASE:
5173	case VMX_VMCS_GUEST_TR_BASE:
5174	case VMX_VMCS_GUEST_GDTR_BASE:
5175	case VMX_VMCS_GUEST_IDTR_BASE:
5176	case VMX_VMCS_GUEST_RSP:
5177	case VMX_VMCS_GUEST_RIP:
5178	case VMX_VMCS_GUEST_SYSENTER_ESP:
5179	case VMX_VMCS_GUEST_SYSENTER_EIP:
5180	{
5181	if (!(u64Val >> 32))
5182	{
5183	/* If this field is 64-bit, VT-x will zero out the top bits. */
5184	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5185	}
5186	else
5187	{
5188	/* Assert that only the 32->64 switcher case should ever come here. */
5189	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5190	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5191	}
5192	break;
5193	}
5194
5195	default:
5196	{
5197	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5198	rc = VERR_INVALID_PARAMETER;
5199	break;
5200	}
5201	}
5202	AssertRCReturn(rc, rc);
5203	return rc;
5204	}
5205
5206
5207	/**
5208	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5209	* hosts (except darwin) for 64-bit guests.
5210	*
5211	* @param pVCpu Pointer to the VMCPU.
5212	* @param idxField The VMCS field encoding.
5213	* @param u64Val 16, 32 or 64-bit value.
5214	*/
5215	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5216	{
5217	AssertPtr(pVCpu);
5218	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5219
5220	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5221	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5222
5223	/* Make sure there are no duplicates. */
5224	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5225	{
5226	if (pCache->Write.aField[i] == idxField)
5227	{
5228	pCache->Write.aFieldVal[i] = u64Val;
5229	return VINF_SUCCESS;
5230	}
5231	}
5232
5233	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5234	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5235	pCache->Write.cValidEntries++;
5236	return VINF_SUCCESS;
5237	}
5238
5239	/* Enable later when the assembly code uses these as callbacks. */
5240	#if 0
5241	/*
5242	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5243	*
5244	* @param pVCpu Pointer to the VMCPU.
5245	* @param pCache Pointer to the VMCS cache.
5246	*
5247	* @remarks No-long-jump zone!!!
5248	*/
5249	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5250	{
5251	AssertPtr(pCache);
5252	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5253	{
5254	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5255	AssertRC(rc);
5256	}
5257	pCache->Write.cValidEntries = 0;
5258	}
5259
5260
5261	/**
5262	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5263	*
5264	* @param pVCpu Pointer to the VMCPU.
5265	* @param pCache Pointer to the VMCS cache.
5266	*
5267	* @remarks No-long-jump zone!!!
5268	*/
5269	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5270	{
5271	AssertPtr(pCache);
5272	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5273	{
5274	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5275	AssertRC(rc);
5276	}
5277	}
5278	#endif
5279	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5280
5281
5282	/**
5283	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5284	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5285	* timer.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	*
5290	* @remarks No-long-jump zone!!!
5291	*/
5292	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5293	{
5294	int rc = VERR_INTERNAL_ERROR_5;
5295	bool fOffsettedTsc = false;
5296	PVM pVM = pVCpu->CTX_SUFF(pVM);
5297	if (pVM->hm.s.vmx.fUsePreemptTimer)
5298	{
5299	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5300
5301	/* Make sure the returned values have sane upper and lower boundaries. */
5302	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5303	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5304	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5305	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5306
5307	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5308	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5309	}
5310	else
5311	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5312
5313	if (fOffsettedTsc)
5314	{
5315	uint64_t u64CurTSC = ASMReadTSC();
5316	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5317	{
5318	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5319	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5320
5321	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5322	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5323	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5324	}
5325	else
5326	{
5327	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5329	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5331	}
5332	}
5333	else
5334	{
5335	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5336	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5339	}
5340	}
5341
5342
5343	/**
5344	* Determines if an exception is a contributory exception. Contributory
5345	* exceptions are ones which can cause double-faults. Page-fault is
5346	* intentionally not included here as it's a conditional contributory exception.
5347	*
5348	* @returns true if the exception is contributory, false otherwise.
5349	* @param uVector The exception vector.
5350	*/
5351	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5352	{
5353	switch (uVector)
5354	{
5355	case X86_XCPT_GP:
5356	case X86_XCPT_SS:
5357	case X86_XCPT_NP:
5358	case X86_XCPT_TS:
5359	case X86_XCPT_DE:
5360	return true;
5361	default:
5362	break;
5363	}
5364	return false;
5365	}
5366
5367
5368	/**
5369	* Sets an event as a pending event to be injected into the guest.
5370	*
5371	* @param pVCpu Pointer to the VMCPU.
5372	* @param u32IntInfo The VM-entry interruption-information field.
5373	* @param cbInstr The VM-entry instruction length in bytes (for software
5374	* interrupts, exceptions and privileged software
5375	* exceptions).
5376	* @param u32ErrCode The VM-entry exception error code.
5377	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5378	* page-fault.
5379	*
5380	* @remarks Statistics counter assumes this is a guest event being injected or
5381	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5382	* always incremented.
5383	*/
5384	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5385	RTGCUINTPTR GCPtrFaultAddress)
5386	{
5387	Assert(!pVCpu->hm.s.Event.fPending);
5388	pVCpu->hm.s.Event.fPending = true;
5389	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5390	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5391	pVCpu->hm.s.Event.cbInstr = cbInstr;
5392	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5393
5394	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5395	}
5396
5397
5398	/**
5399	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5400	*
5401	* @param pVCpu Pointer to the VMCPU.
5402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5403	* out-of-sync. Make sure to update the required fields
5404	* before using them.
5405	*/
5406	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5407	{
5408	NOREF(pMixedCtx);
5409	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5410	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5411	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5412	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5413	}
5414
5415
5416	/**
5417	* Handle a condition that occurred while delivering an event through the guest
5418	* IDT.
5419	*
5420	* @returns VBox status code (informational error codes included).
5421	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5422	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5423	* continue execution of the guest which will delivery the #DF.
5424	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5425	*
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	* @param pVmxTransient Pointer to the VMX transient structure.
5431	*
5432	* @remarks No-long-jump zone!!!
5433	*/
5434	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5435	{
5436	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5437	AssertRCReturn(rc, rc);
5438	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5439	{
5440	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5441	AssertRCReturn(rc, rc);
5442
5443	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5444	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5445	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5446
5447	typedef enum
5448	{
5449	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5450	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5451	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5452	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5453	} VMXREFLECTXCPT;
5454
5455	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5456	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5457	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5458	{
5459	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5460	{
5461	enmReflect = VMXREFLECTXCPT_XCPT;
5462	#ifdef VBOX_STRICT
5463	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5464	&& uExitVector == X86_XCPT_PF)
5465	{
5466	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5467	}
5468	#endif
5469	if ( uExitVector == X86_XCPT_PF
5470	&& uIdtVector == X86_XCPT_PF)
5471	{
5472	pVmxTransient->fVectoringPF = true;
5473	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5474	}
5475	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5476	&& hmR0VmxIsContributoryXcpt(uExitVector)
5477	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5478	\|\| uIdtVector == X86_XCPT_PF))
5479	{
5480	enmReflect = VMXREFLECTXCPT_DF;
5481	}
5482	else if (uIdtVector == X86_XCPT_DF)
5483	enmReflect = VMXREFLECTXCPT_TF;
5484	}
5485	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5486	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5487	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5488	{
5489	/*
5490	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5491	* (whatever they are) as they reoccur when restarting the instruction.
5492	*/
5493	enmReflect = VMXREFLECTXCPT_XCPT;
5494	}
5495	}
5496	else
5497	{
5498	/*
5499	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5500	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5501	* original exception to the guest after handling the VM-exit.
5502	*/
5503	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5504	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5506	{
5507	enmReflect = VMXREFLECTXCPT_XCPT;
5508	}
5509	}
5510
5511	switch (enmReflect)
5512	{
5513	case VMXREFLECTXCPT_XCPT:
5514	{
5515	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5516	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5517	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5518
5519	uint32_t u32ErrCode = 0;
5520	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5521	{
5522	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5523	AssertRCReturn(rc, rc);
5524	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5525	}
5526
5527	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5528	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5529	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5530	rc = VINF_SUCCESS;
5531	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5532	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5533
5534	break;
5535	}
5536
5537	case VMXREFLECTXCPT_DF:
5538	{
5539	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5540	rc = VINF_HM_DOUBLE_FAULT;
5541	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5542	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5543
5544	break;
5545	}
5546
5547	case VMXREFLECTXCPT_TF:
5548	{
5549	rc = VINF_EM_RESET;
5550	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5551	uExitVector));
5552	break;
5553	}
5554
5555	default:
5556	Assert(rc == VINF_SUCCESS);
5557	break;
5558	}
5559	}
5560	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5561	return rc;
5562	}
5563
5564
5565	/**
5566	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5567	*
5568	* @returns VBox status code.
5569	* @param pVCpu Pointer to the VMCPU.
5570	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5571	* out-of-sync. Make sure to update the required fields
5572	* before using them.
5573	*
5574	* @remarks No-long-jump zone!!!
5575	*/
5576	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5577	{
5578	NOREF(pMixedCtx);
5579
5580	/* Since this can be called from our preemption hook it's safer to make the guest-CR0 update non-preemptible. */
5581	VMMRZCallRing3Disable(pVCpu);
5582	HM_DISABLE_PREEMPT_IF_NEEDED();
5583
5584	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5585	{
5586	uint32_t uVal = 0;
5587	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5588	AssertRCReturn(rc, rc);
5589
5590	uint32_t uShadow = 0;
5591	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5592	AssertRCReturn(rc, rc);
5593
5594	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5595	CPUMSetGuestCR0(pVCpu, uVal);
5596	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5597	}
5598
5599	HM_RESTORE_PREEMPT_IF_NEEDED();
5600	VMMRZCallRing3Enable(pVCpu);
5601
5602	return VINF_SUCCESS;
5603	}
5604
5605
5606	/**
5607	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5608	*
5609	* @returns VBox status code.
5610	* @param pVCpu Pointer to the VMCPU.
5611	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5612	* out-of-sync. Make sure to update the required fields
5613	* before using them.
5614	*
5615	* @remarks No-long-jump zone!!!
5616	*/
5617	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5618	{
5619	NOREF(pMixedCtx);
5620
5621	int rc = VINF_SUCCESS;
5622	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5623	{
5624	uint32_t uVal = 0;
5625	uint32_t uShadow = 0;
5626	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5627	AssertRCReturn(rc, rc);
5628	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5629	AssertRCReturn(rc, rc);
5630
5631	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5632	CPUMSetGuestCR4(pVCpu, uVal);
5633	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5634	}
5635	return rc;
5636	}
5637
5638
5639	/**
5640	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5641	*
5642	* @returns VBox status code.
5643	* @param pVCpu Pointer to the VMCPU.
5644	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5645	* out-of-sync. Make sure to update the required fields
5646	* before using them.
5647	*
5648	* @remarks No-long-jump zone!!!
5649	*/
5650	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5651	{
5652	int rc = VINF_SUCCESS;
5653	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5654	{
5655	uint64_t u64Val = 0;
5656	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5657	AssertRCReturn(rc, rc);
5658
5659	pMixedCtx->rip = u64Val;
5660	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5661	}
5662	return rc;
5663	}
5664
5665
5666	/**
5667	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5668	*
5669	* @returns VBox status code.
5670	* @param pVCpu Pointer to the VMCPU.
5671	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5672	* out-of-sync. Make sure to update the required fields
5673	* before using them.
5674	*
5675	* @remarks No-long-jump zone!!!
5676	*/
5677	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5678	{
5679	int rc = VINF_SUCCESS;
5680	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5681	{
5682	uint64_t u64Val = 0;
5683	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5684	AssertRCReturn(rc, rc);
5685
5686	pMixedCtx->rsp = u64Val;
5687	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5688	}
5689	return rc;
5690	}
5691
5692
5693	/**
5694	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5695	*
5696	* @returns VBox status code.
5697	* @param pVCpu Pointer to the VMCPU.
5698	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5699	* out-of-sync. Make sure to update the required fields
5700	* before using them.
5701	*
5702	* @remarks No-long-jump zone!!!
5703	*/
5704	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5707	{
5708	uint32_t uVal = 0;
5709	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5710	AssertRCReturn(rc, rc);
5711
5712	pMixedCtx->eflags.u32 = uVal;
5713	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5714	{
5715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5716	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5717
5718	pMixedCtx->eflags.Bits.u1VM = 0;
5719	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5720	}
5721
5722	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5723	}
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	/**
5729	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5730	* guest-CPU context.
5731	*/
5732	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5733	{
5734	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5735	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5736	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5737	return rc;
5738	}
5739
5740
5741	/**
5742	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5743	* from the guest-state area in the VMCS.
5744	*
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	uint32_t uIntrState = 0;
5755	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5756	AssertRC(rc);
5757
5758	if (!uIntrState)
5759	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5760	else
5761	{
5762	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5763	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5764	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5765	AssertRC(rc);
5766	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5767	AssertRC(rc);
5768
5769	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5770	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5771	}
5772	}
5773
5774
5775	/**
5776	* Saves the guest's activity state.
5777	*
5778	* @returns VBox status code.
5779	* @param pVCpu Pointer to the VMCPU.
5780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781	* out-of-sync. Make sure to update the required fields
5782	* before using them.
5783	*
5784	* @remarks No-long-jump zone!!!
5785	*/
5786	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787	{
5788	NOREF(pMixedCtx);
5789	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5790	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5791	return VINF_SUCCESS;
5792	}
5793
5794
5795	/**
5796	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5797	* the current VMCS into the guest-CPU context.
5798	*
5799	* @returns VBox status code.
5800	* @param pVCpu Pointer to the VMCPU.
5801	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5802	* out-of-sync. Make sure to update the required fields
5803	* before using them.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5808	{
5809	int rc = VINF_SUCCESS;
5810	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5811	{
5812	uint32_t u32Val = 0;
5813	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5814	pMixedCtx->SysEnter.cs = u32Val;
5815	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5816	}
5817
5818	uint64_t u64Val = 0;
5819	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5822	pMixedCtx->SysEnter.eip = u64Val;
5823	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5824	}
5825	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5826	{
5827	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5828	pMixedCtx->SysEnter.esp = u64Val;
5829	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5830	}
5831	return rc;
5832	}
5833
5834
5835	/**
5836	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5837	* the CPU back into the guest-CPU context.
5838	*
5839	* @returns VBox status code.
5840	* @param pVCpu Pointer to the VMCPU.
5841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5842	* out-of-sync. Make sure to update the required fields
5843	* before using them.
5844	*
5845	* @remarks No-long-jump zone!!!
5846	*/
5847	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5848	{
5849	#if HC_ARCH_BITS == 64
5850	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5851	{
5852	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5853	VMMRZCallRing3Disable(pVCpu);
5854	HM_DISABLE_PREEMPT_IF_NEEDED();
5855
5856	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5857	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5858	{
5859	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5860	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5861	}
5862
5863	HM_RESTORE_PREEMPT_IF_NEEDED();
5864	VMMRZCallRing3Enable(pVCpu);
5865	}
5866	else
5867	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5868	#else
5869	NOREF(pMixedCtx);
5870	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5871	#endif
5872
5873	return VINF_SUCCESS;
5874	}
5875
5876
5877	/**
5878	* Saves the auto load/store'd guest MSRs from the current VMCS into
5879	* the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5892	return VINF_SUCCESS;
5893
5894	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5895	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5896	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5897	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5898	{
5899	switch (pMsr->u32Msr)
5900	{
5901	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5902	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5903	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5904	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5905	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5906	default:
5907	{
5908	AssertFailed();
5909	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5910	}
5911	}
5912	}
5913
5914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5915	return VINF_SUCCESS;
5916	}
5917
5918
5919	/**
5920	* Saves the guest control registers from the current VMCS into the guest-CPU
5921	* context.
5922	*
5923	* @returns VBox status code.
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	*
5929	* @remarks No-long-jump zone!!!
5930	*/
5931	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5932	{
5933	/* Guest CR0. Guest FPU. */
5934	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5935	AssertRCReturn(rc, rc);
5936
5937	/* Guest CR4. */
5938	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5939	AssertRCReturn(rc, rc);
5940
5941	/* Guest CR2 - updated always during the world-switch or in #PF. */
5942	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5943	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5944	{
5945	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5946	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5947
5948	PVM pVM = pVCpu->CTX_SUFF(pVM);
5949	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5950	\|\| ( pVM->hm.s.fNestedPaging
5951	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5952	{
5953	uint64_t u64Val = 0;
5954	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5955	if (pMixedCtx->cr3 != u64Val)
5956	{
5957	CPUMSetGuestCR3(pVCpu, u64Val);
5958	if (VMMRZCallRing3IsEnabled(pVCpu))
5959	{
5960	PGMUpdateCR3(pVCpu, u64Val);
5961	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5962	}
5963	else
5964	{
5965	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5966	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5967	}
5968	}
5969
5970	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5971	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5972	{
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5977
5978	if (VMMRZCallRing3IsEnabled(pVCpu))
5979	{
5980	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5982	}
5983	else
5984	{
5985	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5986	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5987	}
5988	}
5989	}
5990
5991	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
5992	}
5993
5994	/*
5995	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5996	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5997	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5998	*
5999	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6000	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6001	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6002	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6003	*
6004	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6005	*/
6006	if (VMMRZCallRing3IsEnabled(pVCpu))
6007	{
6008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6009	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6010
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6013
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6015	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6016	}
6017
6018	return rc;
6019	}
6020
6021
6022	/**
6023	* Reads a guest segment register from the current VMCS into the guest-CPU
6024	* context.
6025	*
6026	* @returns VBox status code.
6027	* @param pVCpu Pointer to the VMCPU.
6028	* @param idxSel Index of the selector in the VMCS.
6029	* @param idxLimit Index of the segment limit in the VMCS.
6030	* @param idxBase Index of the segment base in the VMCS.
6031	* @param idxAccess Index of the access rights of the segment in the VMCS.
6032	* @param pSelReg Pointer to the segment selector.
6033	*
6034	* @remarks No-long-jump zone!!!
6035	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6036	* macro as that takes care of whether to read from the VMCS cache or
6037	* not.
6038	*/
6039	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6040	PCPUMSELREG pSelReg)
6041	{
6042	NOREF(pVCpu);
6043
6044	uint32_t u32Val = 0;
6045	int rc = VMXReadVmcs32(idxSel, &u32Val);
6046	AssertRCReturn(rc, rc);
6047	pSelReg->Sel = (uint16_t)u32Val;
6048	pSelReg->ValidSel = (uint16_t)u32Val;
6049	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6050
6051	rc = VMXReadVmcs32(idxLimit, &u32Val);
6052	AssertRCReturn(rc, rc);
6053	pSelReg->u32Limit = u32Val;
6054
6055	uint64_t u64Val = 0;
6056	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6057	AssertRCReturn(rc, rc);
6058	pSelReg->u64Base = u64Val;
6059
6060	rc = VMXReadVmcs32(idxAccess, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->Attr.u = u32Val;
6063
6064	/*
6065	* If VT-x marks the segment as unusable, most other bits remain undefined:
6066	* - For CS the L, D and G bits have meaning.
6067	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6068	* - For the remaining data segments no bits are defined.
6069	*
6070	* The present bit and the unusable bit has been observed to be set at the
6071	* same time (the selector was supposed to invalid as we started executing
6072	* a V8086 interrupt in ring-0).
6073	*
6074	* What should be important for the rest of the VBox code, is that the P bit is
6075	* cleared. Some of the other VBox code recognizes the unusable bit, but
6076	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6077	* safe side here, we'll strip off P and other bits we don't care about. If
6078	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6079	*
6080	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6081	*/
6082	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6083	{
6084	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6085
6086	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6087	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6088	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6089
6090	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6091	#ifdef DEBUG_bird
6092	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6093	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6094	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6095	#endif
6096	}
6097	return VINF_SUCCESS;
6098	}
6099
6100
6101	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6102	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6103	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6104	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6105	#else
6106	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6107	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6108	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6109	#endif
6110
6111
6112	/**
6113	* Saves the guest segment registers from the current VMCS into the guest-CPU
6114	* context.
6115	*
6116	* @returns VBox status code.
6117	* @param pVCpu Pointer to the VMCPU.
6118	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6119	* out-of-sync. Make sure to update the required fields
6120	* before using them.
6121	*
6122	* @remarks No-long-jump zone!!!
6123	*/
6124	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6125	{
6126	/* Guest segment registers. */
6127	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6128	{
6129	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6135	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6136
6137	/* Restore segment attributes for real-on-v86 mode hack. */
6138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6139	{
6140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6146	}
6147	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6148	}
6149
6150	return VINF_SUCCESS;
6151	}
6152
6153
6154	/**
6155	* Saves the guest descriptor table registers and task register from the current
6156	* VMCS into the guest-CPU context.
6157	*
6158	* @returns VBox status code.
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6161	* out-of-sync. Make sure to update the required fields
6162	* before using them.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6167	{
6168	int rc = VINF_SUCCESS;
6169
6170	/* Guest LDTR. */
6171	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6172	{
6173	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6174	AssertRCReturn(rc, rc);
6175	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6176	}
6177
6178	/* Guest GDTR. */
6179	uint64_t u64Val = 0;
6180	uint32_t u32Val = 0;
6181	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6182	{
6183	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6185	pMixedCtx->gdtr.pGdt = u64Val;
6186	pMixedCtx->gdtr.cbGdt = u32Val;
6187	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6188	}
6189
6190	/* Guest IDTR. */
6191	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6192	{
6193	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6194	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6195	pMixedCtx->idtr.pIdt = u64Val;
6196	pMixedCtx->idtr.cbIdt = u32Val;
6197	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6198	}
6199
6200	/* Guest TR. */
6201	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6202	{
6203	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6204	AssertRCReturn(rc, rc);
6205
6206	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6207	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208	{
6209	rc = VMXLOCAL_READ_SEG(TR, tr);
6210	AssertRCReturn(rc, rc);
6211	}
6212	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6213	}
6214	return rc;
6215	}
6216
6217	#undef VMXLOCAL_READ_SEG
6218
6219
6220	/**
6221	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6222	* context.
6223	*
6224	* @returns VBox status code.
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	*
6230	* @remarks No-long-jump zone!!!
6231	*/
6232	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6233	{
6234	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6235	{
6236	if (!pVCpu->hm.s.fUsingHyperDR7)
6237	{
6238	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6239	uint32_t u32Val;
6240	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6241	pMixedCtx->dr[7] = u32Val;
6242	}
6243
6244	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6245	}
6246	return VINF_SUCCESS;
6247	}
6248
6249
6250	/**
6251	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6252	*
6253	* @returns VBox status code.
6254	* @param pVCpu Pointer to the VMCPU.
6255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256	* out-of-sync. Make sure to update the required fields
6257	* before using them.
6258	*
6259	* @remarks No-long-jump zone!!!
6260	*/
6261	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262	{
6263	NOREF(pMixedCtx);
6264
6265	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6267	return VINF_SUCCESS;
6268	}
6269
6270
6271	/**
6272	* Saves the entire guest state from the currently active VMCS into the
6273	* guest-CPU context. This essentially VMREADs all guest-data.
6274	*
6275	* @returns VBox status code.
6276	* @param pVCpu Pointer to the VMCPU.
6277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6278	* out-of-sync. Make sure to update the required fields
6279	* before using them.
6280	*/
6281	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	Assert(pVCpu);
6284	Assert(pMixedCtx);
6285
6286	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6287	return VINF_SUCCESS;
6288
6289	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6290	again on the ring-3 callback path, there is no real need to. */
6291	if (VMMRZCallRing3IsEnabled(pVCpu))
6292	VMMR0LogFlushDisable(pVCpu);
6293	else
6294	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6295	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6296
6297	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6328	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6329
6330	if (VMMRZCallRing3IsEnabled(pVCpu))
6331	VMMR0LogFlushEnable(pVCpu);
6332
6333	return rc;
6334	}
6335
6336
6337	/**
6338	* Check per-VM and per-VCPU force flag actions that require us to go back to
6339	* ring-3 for one reason or another.
6340	*
6341	* @returns VBox status code (information status code included).
6342	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6343	* ring-3.
6344	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6345	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6346	* interrupts)
6347	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6348	* all EMTs to be in ring-3.
6349	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6350	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6351	* to the EM loop.
6352	*
6353	* @param pVM Pointer to the VM.
6354	* @param pVCpu Pointer to the VMCPU.
6355	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6356	* out-of-sync. Make sure to update the required fields
6357	* before using them.
6358	*/
6359	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6360	{
6361	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6362
6363	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6364	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6365	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6366	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6367	{
6368	/* We need the control registers now, make sure the guest-CPU context is updated. */
6369	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6370	AssertRCReturn(rc3, rc3);
6371
6372	/* Pending HM CR3 sync. */
6373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6374	{
6375	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6376	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6377	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6378	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6379	}
6380
6381	/* Pending HM PAE PDPEs. */
6382	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6383	{
6384	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6385	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6386	}
6387
6388	/* Pending PGM C3 sync. */
6389	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6390	{
6391	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6392	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6393	if (rc2 != VINF_SUCCESS)
6394	{
6395	AssertRC(rc2);
6396	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6397	return rc2;
6398	}
6399	}
6400
6401	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6402	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6403	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6404	{
6405	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6406	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6407	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6408	return rc2;
6409	}
6410
6411	/* Pending VM request packets, such as hardware interrupts. */
6412	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6413	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6414	{
6415	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6416	return VINF_EM_PENDING_REQUEST;
6417	}
6418
6419	/* Pending PGM pool flushes. */
6420	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6421	{
6422	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6423	return VINF_PGM_POOL_FLUSH_PENDING;
6424	}
6425
6426	/* Pending DMA requests. */
6427	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6428	{
6429	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6430	return VINF_EM_RAW_TO_R3;
6431	}
6432	}
6433
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Converts any TRPM trap into a pending HM event. This is typically used when
6440	* entering from ring-3 (not longjmp returns).
6441	*
6442	* @param pVCpu Pointer to the VMCPU.
6443	*/
6444	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6445	{
6446	Assert(TRPMHasTrap(pVCpu));
6447	Assert(!pVCpu->hm.s.Event.fPending);
6448
6449	uint8_t uVector;
6450	TRPMEVENT enmTrpmEvent;
6451	RTGCUINT uErrCode;
6452	RTGCUINTPTR GCPtrFaultAddress;
6453	uint8_t cbInstr;
6454
6455	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6456	AssertRC(rc);
6457
6458	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6459	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6460	if (enmTrpmEvent == TRPM_TRAP)
6461	{
6462	switch (uVector)
6463	{
6464	case X86_XCPT_BP:
6465	case X86_XCPT_OF:
6466	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6467	break;
6468
6469	case X86_XCPT_PF:
6470	case X86_XCPT_DF:
6471	case X86_XCPT_TS:
6472	case X86_XCPT_NP:
6473	case X86_XCPT_SS:
6474	case X86_XCPT_GP:
6475	case X86_XCPT_AC:
6476	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6477	/* no break! */
6478	default:
6479	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6480	break;
6481	}
6482	}
6483	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6484	{
6485	if (uVector == X86_XCPT_NMI)
6486	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	else
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	}
6490	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6491	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6492	else
6493	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6494
6495	rc = TRPMResetTrap(pVCpu);
6496	AssertRC(rc);
6497	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6498	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6499
6500	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6501	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6502	}
6503
6504
6505	/**
6506	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6507	* VT-x to execute any instruction.
6508	*
6509	* @param pvCpu Pointer to the VMCPU.
6510	*/
6511	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6512	{
6513	Assert(pVCpu->hm.s.Event.fPending);
6514
6515	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6516	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6517	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6518	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6519
6520	/* If a trap was already pending, we did something wrong! */
6521	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6522
6523	TRPMEVENT enmTrapType;
6524	switch (uVectorType)
6525	{
6526	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6527	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6528	enmTrapType = TRPM_HARDWARE_INT;
6529	break;
6530
6531	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6532	enmTrapType = TRPM_SOFTWARE_INT;
6533	break;
6534
6535	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6536	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6538	enmTrapType = TRPM_TRAP;
6539	break;
6540
6541	default:
6542	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6543	enmTrapType = TRPM_32BIT_HACK;
6544	break;
6545	}
6546
6547	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6548
6549	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6550	AssertRC(rc);
6551
6552	if (fErrorCodeValid)
6553	TRPMSetErrorCode(pVCpu, uErrorCode);
6554
6555	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6556	&& uVector == X86_XCPT_PF)
6557	{
6558	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6559	}
6560	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6561	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6562	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6563	{
6564	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6565	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6566	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6567	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6568	}
6569	pVCpu->hm.s.Event.fPending = false;
6570	}
6571
6572
6573	/**
6574	* Does the necessary state syncing before returning to ring-3 for any reason
6575	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6576	*
6577	* @returns VBox status code.
6578	* @param pVM Pointer to the VM.
6579	* @param pVCpu Pointer to the VMCPU.
6580	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6581	* be out-of-sync. Make sure to update the required
6582	* fields before using them.
6583	* @param fSaveGuestState Whether to save the guest state or not.
6584	*
6585	* @remarks No-long-jmp zone!!!
6586	*/
6587	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6588	{
6589	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6590	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6591
6592	RTCPUID idCpu = RTMpCpuId();
6593	Log4Func(("HostCpuId=%u\n", idCpu));
6594
6595	/*
6596	* IMPORTANT!!!
6597	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
6598	*/
6599
6600	/* Save the guest state if necessary. */
6601	if ( fSaveGuestState
6602	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6603	{
6604	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6605	AssertRCReturn(rc, rc);
6606	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6607	}
6608
6609	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6610	if (CPUMIsGuestFPUStateActive(pVCpu))
6611	{
6612	/* We shouldn't reload CR0 without saving it first. */
6613	if (!fSaveGuestState)
6614	{
6615	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6616	AssertRCReturn(rc, rc);
6617	}
6618	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6619	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6620	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6621	}
6622
6623	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6624	#ifdef VBOX_STRICT
6625	if (CPUMIsHyperDebugStateActive(pVCpu))
6626	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6627	#endif
6628	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6629	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6630	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6631	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6632
6633	#if HC_ARCH_BITS == 64
6634	/* Restore host-state bits that VT-x only restores partially. */
6635	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6636	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6637	{
6638	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6639	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6640	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6641	}
6642	#endif
6643
6644	#if HC_ARCH_BITS == 64
6645	/* Restore the host MSRs as we're leaving VT-x context. */
6646	if ( pVM->hm.s.fAllow64BitGuests
6647	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6648	{
6649	/* We shouldn't reload the guest MSRs without saving it first. */
6650	if (!fSaveGuestState)
6651	{
6652	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6653	AssertRCReturn(rc, rc);
6654	}
6655	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6656	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6657	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6658	}
6659	#endif
6660
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6665	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6666	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6667	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6669
6670	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6671
6672	/** @todo This partially defeats the purpose of having preemption hooks.
6673	* The problem is, deregistering the hooks should be moved to a place that
6674	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6675	* context.
6676	*/
6677	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6678	{
6679	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6680	AssertRCReturn(rc, rc);
6681
6682	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6683	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6684	}
6685	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6686	NOREF(idCpu);
6687
6688	return VINF_SUCCESS;
6689	}
6690
6691
6692	/**
6693	* Leaves the VT-x session.
6694	*
6695	* @returns VBox status code.
6696	* @param pVM Pointer to the VM.
6697	* @param pVCpu Pointer to the VMCPU.
6698	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6699	* out-of-sync. Make sure to update the required fields
6700	* before using them.
6701	*
6702	* @remarks No-long-jmp zone!!!
6703	*/
6704	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6705	{
6706	HM_DISABLE_PREEMPT_IF_NEEDED();
6707	HMVMX_ASSERT_CPU_SAFE();
6708	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6709	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6710
6711	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6712	and done this from the VMXR0ThreadCtxCallback(). */
6713	if (!pVCpu->hm.s.fLeaveDone)
6714	{
6715	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6716	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6717	pVCpu->hm.s.fLeaveDone = true;
6718	}
6719
6720	/*
6721	* IMPORTANT!!!
6722	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
6723	*/
6724
6725	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6726	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6727	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6728	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6729	VMMR0ThreadCtxHooksDeregister(pVCpu);
6730
6731	/* Leave HM context. This takes care of local init (term). */
6732	int rc = HMR0LeaveCpu(pVCpu);
6733
6734	HM_RESTORE_PREEMPT_IF_NEEDED();
6735
6736	return rc;
6737	}
6738
6739
6740	/**
6741	* Does the necessary state syncing before doing a longjmp to ring-3.
6742	*
6743	* @returns VBox status code.
6744	* @param pVM Pointer to the VM.
6745	* @param pVCpu Pointer to the VMCPU.
6746	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6747	* out-of-sync. Make sure to update the required fields
6748	* before using them.
6749	*
6750	* @remarks No-long-jmp zone!!!
6751	*/
6752	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6753	{
6754	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6755	}
6756
6757
6758	/**
6759	* Take necessary actions before going back to ring-3.
6760	*
6761	* An action requires us to go back to ring-3. This function does the necessary
6762	* steps before we can safely return to ring-3. This is not the same as longjmps
6763	* to ring-3, this is voluntary and prepares the guest so it may continue
6764	* executing outside HM (recompiler/IEM).
6765	*
6766	* @returns VBox status code.
6767	* @param pVM Pointer to the VM.
6768	* @param pVCpu Pointer to the VMCPU.
6769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6770	* out-of-sync. Make sure to update the required fields
6771	* before using them.
6772	* @param rcExit The reason for exiting to ring-3. Can be
6773	* VINF_VMM_UNKNOWN_RING3_CALL.
6774	*/
6775	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6776	{
6777	Assert(pVM);
6778	Assert(pVCpu);
6779	Assert(pMixedCtx);
6780	HMVMX_ASSERT_PREEMPT_SAFE();
6781
6782	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6783	{
6784	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6785	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6786	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6787	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6788	}
6789
6790	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6791	VMMRZCallRing3Disable(pVCpu);
6792	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6793
6794	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6795	if (pVCpu->hm.s.Event.fPending)
6796	{
6797	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6798	Assert(!pVCpu->hm.s.Event.fPending);
6799	}
6800
6801	/* Save guest state and restore host state bits. */
6802	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6803	AssertRCReturn(rc, rc);
6804	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6805
6806	/* Sync recompiler state. */
6807	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6808	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6809	\| CPUM_CHANGED_LDTR
6810	\| CPUM_CHANGED_GDTR
6811	\| CPUM_CHANGED_IDTR
6812	\| CPUM_CHANGED_TR
6813	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6814	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6815	if ( pVM->hm.s.fNestedPaging
6816	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6817	{
6818	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6819	}
6820
6821	Assert(!pVCpu->hm.s.fClearTrapFlag);
6822
6823	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6824	if (rcExit != VINF_EM_RAW_INTERRUPT)
6825	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6826
6827	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6828
6829	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6830	VMMRZCallRing3RemoveNotification(pVCpu);
6831	VMMRZCallRing3Enable(pVCpu);
6832
6833	return rc;
6834	}
6835
6836
6837	/**
6838	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6839	* longjump to ring-3 and possibly get preempted.
6840	*
6841	* @returns VBox status code.
6842	* @param pVCpu Pointer to the VMCPU.
6843	* @param enmOperation The operation causing the ring-3 longjump.
6844	* @param pvUser Opaque pointer to the guest-CPU context. The data
6845	* may be out-of-sync. Make sure to update the required
6846	* fields before using them.
6847	*/
6848	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6849	{
6850	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6851	{
6852	/*
6853	* !!! IMPORTANT !!!
6854	* If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
6855	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
6856	*/
6857	VMMRZCallRing3RemoveNotification(pVCpu);
6858	VMMRZCallRing3Disable(pVCpu);
6859	HM_DISABLE_PREEMPT_IF_NEEDED();
6860
6861	PVM pVM = pVCpu->CTX_SUFF(pVM);
6862	if (CPUMIsGuestFPUStateActive(pVCpu))
6863	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6864
6865	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6866
6867	#if HC_ARCH_BITS == 64
6868	/* Restore host-state bits that VT-x only restores partially. */
6869	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6870	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6871	{
6872	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6873	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6874	}
6875	#endif
6876
6877	#if HC_ARCH_BITS == 64
6878	/* Restore the host MSRs as we're leaving VT-x context. */
6879	if ( pVM->hm.s.fAllow64BitGuests
6880	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6881	{
6882	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6883	}
6884	#endif
6885	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6886	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6887	{
6888	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6889	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6890	}
6891
6892	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6893	VMMR0ThreadCtxHooksDeregister(pVCpu);
6894
6895	HMR0LeaveCpu(pVCpu);
6896	HM_RESTORE_PREEMPT_IF_NEEDED();
6897	return VINF_SUCCESS;
6898	}
6899
6900	Assert(pVCpu);
6901	Assert(pvUser);
6902	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6903	HMVMX_ASSERT_PREEMPT_SAFE();
6904
6905	VMMRZCallRing3Disable(pVCpu);
6906	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6907
6908	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6909	enmOperation));
6910
6911	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6912	AssertRCReturn(rc, rc);
6913
6914	VMMRZCallRing3Enable(pVCpu);
6915	return VINF_SUCCESS;
6916	}
6917
6918
6919	/**
6920	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6921	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6922	*
6923	* @param pVCpu Pointer to the VMCPU.
6924	*/
6925	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6926	{
6927	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6928	{
6929	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6930	{
6931	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6932	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6933	AssertRC(rc);
6934	}
6935	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6936	}
6937
6938
6939	/**
6940	* Evaluates the event to be delivered to the guest and sets it as the pending
6941	* event.
6942	*
6943	* @param pVCpu Pointer to the VMCPU.
6944	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6945	* out-of-sync. Make sure to update the required fields
6946	* before using them.
6947	*/
6948	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6949	{
6950	Assert(!pVCpu->hm.s.Event.fPending);
6951
6952	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6953	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6954	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6955	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6956
6957	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6958	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6959	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6960	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6961	Assert(!TRPMHasTrap(pVCpu));
6962
6963	/** @todo SMI. SMIs take priority over NMIs. */
6964	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6965	{
6966	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6967	if ( !fBlockMovSS
6968	&& !fBlockSti)
6969	{
6970	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6971	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6972	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6973	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6974
6975	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6976	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6977	}
6978	else
6979	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6980	}
6981	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6982	&& !pVCpu->hm.s.fSingleInstruction)
6983	{
6984	/*
6985	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6986	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6987	* evaluated here and not set as pending, solely based on the force-flags.
6988	*/
6989	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6990	AssertRC(rc);
6991	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6992	if ( !fBlockInt
6993	&& !fBlockSti
6994	&& !fBlockMovSS)
6995	{
6996	uint8_t u8Interrupt;
6997	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6998	if (RT_SUCCESS(rc))
6999	{
7000	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7001	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7002	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7003
7004	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7005	}
7006	else
7007	{
7008	/** @todo Does this actually happen? If not turn it into an assertion. */
7009	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7010	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7011	}
7012	}
7013	else
7014	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7015	}
7016	}
7017
7018
7019	/**
7020	* Injects any pending events into the guest if the guest is in a state to
7021	* receive them.
7022	*
7023	* @returns VBox status code (informational status codes included).
7024	* @param pVCpu Pointer to the VMCPU.
7025	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7026	* out-of-sync. Make sure to update the required fields
7027	* before using them.
7028	*/
7029	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7030	{
7031	HMVMX_ASSERT_PREEMPT_SAFE();
7032	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7033
7034	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7035	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7036	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7037	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7038
7039	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7040	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7041	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7042	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7043	Assert(!TRPMHasTrap(pVCpu));
7044
7045	int rc = VINF_SUCCESS;
7046	if (pVCpu->hm.s.Event.fPending)
7047	{
7048	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7049	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7050	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7051	{
7052	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7053	AssertRCReturn(rc, rc);
7054	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7055	Assert(!fBlockInt);
7056	Assert(!fBlockSti);
7057	Assert(!fBlockMovSS);
7058	}
7059	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7060	{
7061	Assert(!fBlockSti);
7062	Assert(!fBlockMovSS);
7063	}
7064	#endif
7065	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7066	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7067	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7068	AssertRCReturn(rc, rc);
7069
7070	/* Update the interruptibility-state as it could have been changed by
7071	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7072	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7073	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7074
7075	#ifdef VBOX_WITH_STATISTICS
7076	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7077	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7078	else
7079	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7080	#endif
7081	}
7082
7083	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7084	if ( !pVCpu->hm.s.fSingleInstruction
7085	&& !DBGFIsStepping(pVCpu))
7086	{
7087	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7088	AssertRCReturn(rc2, rc2);
7089	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7090	{
7091	/*
7092	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7093	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7094	* See Intel spec. 27.3.4 "Saving Non-Register State".
7095	*/
7096	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7097	AssertRCReturn(rc2, rc2);
7098	}
7099	}
7100	else
7101	{
7102	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7103	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7104	uIntrState = 0;
7105	}
7106
7107	/*
7108	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7109	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7110	*/
7111	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7112	AssertRC(rc2);
7113
7114	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7115	NOREF(fBlockMovSS); NOREF(fBlockSti);
7116	return rc;
7117	}
7118
7119
7120	/**
7121	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7122	*
7123	* @param pVCpu Pointer to the VMCPU.
7124	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7125	* out-of-sync. Make sure to update the required fields
7126	* before using them.
7127	*/
7128	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7129	{
7130	NOREF(pMixedCtx);
7131	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7132	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7133	}
7134
7135
7136	/**
7137	* Injects a double-fault (#DF) exception into the VM.
7138	*
7139	* @returns VBox status code (informational status code included).
7140	* @param pVCpu Pointer to the VMCPU.
7141	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7142	* out-of-sync. Make sure to update the required fields
7143	* before using them.
7144	*/
7145	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7146	{
7147	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7148	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7149	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7150	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7151	puIntrState);
7152	}
7153
7154
7155	/**
7156	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7157	*
7158	* @param pVCpu Pointer to the VMCPU.
7159	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7160	* out-of-sync. Make sure to update the required fields
7161	* before using them.
7162	*/
7163	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7164	{
7165	NOREF(pMixedCtx);
7166	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7167	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7168	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7169	}
7170
7171
7172	/**
7173	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7174	*
7175	* @param pVCpu Pointer to the VMCPU.
7176	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7177	* out-of-sync. Make sure to update the required fields
7178	* before using them.
7179	* @param cbInstr The value of RIP that is to be pushed on the guest
7180	* stack.
7181	*/
7182	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7183	{
7184	NOREF(pMixedCtx);
7185	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7186	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7187	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7188	}
7189
7190
7191	/**
7192	* Injects a general-protection (#GP) fault into the VM.
7193	*
7194	* @returns VBox status code (informational status code included).
7195	* @param pVCpu Pointer to the VMCPU.
7196	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7197	* out-of-sync. Make sure to update the required fields
7198	* before using them.
7199	* @param u32ErrorCode The error code associated with the #GP.
7200	*/
7201	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7202	uint32_t *puIntrState)
7203	{
7204	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7205	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7206	if (fErrorCodeValid)
7207	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7208	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7209	puIntrState);
7210	}
7211
7212
7213	/**
7214	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7215	*
7216	* @param pVCpu Pointer to the VMCPU.
7217	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7218	* out-of-sync. Make sure to update the required fields
7219	* before using them.
7220	* @param uVector The software interrupt vector number.
7221	* @param cbInstr The value of RIP that is to be pushed on the guest
7222	* stack.
7223	*/
7224	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7225	{
7226	NOREF(pMixedCtx);
7227	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7228	if ( uVector == X86_XCPT_BP
7229	\|\| uVector == X86_XCPT_OF)
7230	{
7231	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7232	}
7233	else
7234	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7235	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7236	}
7237
7238
7239	/**
7240	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7241	* stack.
7242	*
7243	* @returns VBox status code (information status code included).
7244	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7245	* @param pVM Pointer to the VM.
7246	* @param pMixedCtx Pointer to the guest-CPU context.
7247	* @param uValue The value to push to the guest stack.
7248	*/
7249	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7250	{
7251	/*
7252	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7253	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7254	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7255	*/
7256	if (pMixedCtx->sp == 1)
7257	return VINF_EM_RESET;
7258	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7259	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7260	AssertRCReturn(rc, rc);
7261	return rc;
7262	}
7263
7264
7265	/**
7266	* Injects an event into the guest upon VM-entry by updating the relevant fields
7267	* in the VM-entry area in the VMCS.
7268	*
7269	* @returns VBox status code (informational error codes included).
7270	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7271	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7272	*
7273	* @param pVCpu Pointer to the VMCPU.
7274	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7275	* be out-of-sync. Make sure to update the required
7276	* fields before using them.
7277	* @param u64IntInfo The VM-entry interruption-information field.
7278	* @param cbInstr The VM-entry instruction length in bytes (for
7279	* software interrupts, exceptions and privileged
7280	* software exceptions).
7281	* @param u32ErrCode The VM-entry exception error code.
7282	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7283	* @param puIntrState Pointer to the current guest interruptibility-state.
7284	* This interruptibility-state will be updated if
7285	* necessary. This cannot not be NULL.
7286	*
7287	* @remarks Requires CR0!
7288	* @remarks No-long-jump zone!!!
7289	*/
7290	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7291	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7292	{
7293	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7294	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7295	Assert(puIntrState);
7296	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7297
7298	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7299	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7300
7301	#ifdef VBOX_STRICT
7302	/* Validate the error-code-valid bit for hardware exceptions. */
7303	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7304	{
7305	switch (uVector)
7306	{
7307	case X86_XCPT_PF:
7308	case X86_XCPT_DF:
7309	case X86_XCPT_TS:
7310	case X86_XCPT_NP:
7311	case X86_XCPT_SS:
7312	case X86_XCPT_GP:
7313	case X86_XCPT_AC:
7314	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7315	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7316	/* fallthru */
7317	default:
7318	break;
7319	}
7320	}
7321	#endif
7322
7323	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7324	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7325	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7326
7327	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7328
7329	/* We require CR0 to check if the guest is in real-mode. */
7330	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7331	AssertRCReturn(rc, rc);
7332
7333	/*
7334	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7335	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7336	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7337	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7338	*/
7339	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7340	{
7341	PVM pVM = pVCpu->CTX_SUFF(pVM);
7342	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7343	{
7344	Assert(PDMVmmDevHeapIsEnabled(pVM));
7345	Assert(pVM->hm.s.vmx.pRealModeTSS);
7346
7347	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7348	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7349	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7350	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7351	AssertRCReturn(rc, rc);
7352	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7353
7354	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7355	const size_t cbIdtEntry = sizeof(X86IDTR16);
7356	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7357	{
7358	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7359	if (uVector == X86_XCPT_DF)
7360	return VINF_EM_RESET;
7361	else if (uVector == X86_XCPT_GP)
7362	{
7363	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7364	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7365	}
7366
7367	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7368	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7369	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7370	}
7371
7372	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7373	uint16_t uGuestIp = pMixedCtx->ip;
7374	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7375	{
7376	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7377	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7378	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7379	}
7380	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7381	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7382
7383	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7384	X86IDTR16 IdtEntry;
7385	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7386	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7387	AssertRCReturn(rc, rc);
7388
7389	/* Construct the stack frame for the interrupt/exception handler. */
7390	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7391	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7392	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7393	AssertRCReturn(rc, rc);
7394
7395	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7396	if (rc == VINF_SUCCESS)
7397	{
7398	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7399	pMixedCtx->rip = IdtEntry.offSel;
7400	pMixedCtx->cs.Sel = IdtEntry.uSel;
7401	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7402	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7403	&& uVector == X86_XCPT_PF)
7404	{
7405	pMixedCtx->cr2 = GCPtrFaultAddress;
7406	}
7407
7408	/* If any other guest-state bits are changed here, make sure to update
7409	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7410	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7411	\| HM_CHANGED_GUEST_RIP
7412	\| HM_CHANGED_GUEST_RFLAGS
7413	\| HM_CHANGED_GUEST_RSP);
7414
7415	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7416	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7417	{
7418	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7419	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7420	Log4(("Clearing inhibition due to STI.\n"));
7421	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7422	}
7423	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7424
7425	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7426	it, if we are returning to ring-3 before executing guest code. */
7427	pVCpu->hm.s.Event.fPending = false;
7428	}
7429	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7430	return rc;
7431	}
7432	else
7433	{
7434	/*
7435	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7436	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7437	*/
7438	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7439	}
7440	}
7441
7442	/* Validate. */
7443	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7444	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7445	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7446
7447	/* Inject. */
7448	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7449	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7450	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7451	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7452
7453	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7454	&& uVector == X86_XCPT_PF)
7455	{
7456	pMixedCtx->cr2 = GCPtrFaultAddress;
7457	}
7458
7459	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7460	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7461
7462	AssertRCReturn(rc, rc);
7463	return rc;
7464	}
7465
7466
7467	/**
7468	* Clears the interrupt-window exiting control in the VMCS and if necessary
7469	* clears the current event in the VMCS as well.
7470	*
7471	* @returns VBox status code.
7472	* @param pVCpu Pointer to the VMCPU.
7473	*
7474	* @remarks Use this function only to clear events that have not yet been
7475	* delivered to the guest but are injected in the VMCS!
7476	* @remarks No-long-jump zone!!!
7477	*/
7478	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7479	{
7480	int rc;
7481	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7482
7483	/* Clear interrupt-window exiting control. */
7484	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7485	{
7486	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7487	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7488	AssertRC(rc);
7489	}
7490
7491	if (!pVCpu->hm.s.Event.fPending)
7492	return;
7493
7494	#ifdef VBOX_STRICT
7495	uint32_t u32EntryInfo;
7496	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7497	AssertRC(rc);
7498	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7499	#endif
7500
7501	/* Clear the entry-interruption field (including the valid bit). */
7502	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7503	AssertRC(rc);
7504
7505	/* Clear the pending debug exception field. */
7506	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7507	AssertRC(rc);
7508	}
7509
7510
7511	/**
7512	* Enters the VT-x session.
7513	*
7514	* @returns VBox status code.
7515	* @param pVM Pointer to the VM.
7516	* @param pVCpu Pointer to the VMCPU.
7517	* @param pCpu Pointer to the CPU info struct.
7518	*/
7519	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7520	{
7521	AssertPtr(pVM);
7522	AssertPtr(pVCpu);
7523	Assert(pVM->hm.s.vmx.fSupported);
7524	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7525	NOREF(pCpu); NOREF(pVM);
7526
7527	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7528	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7529
7530	#ifdef VBOX_STRICT
7531	/* Make sure we're in VMX root mode. */
7532	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7533	if (!(u32HostCR4 & X86_CR4_VMXE))
7534	{
7535	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7536	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7537	}
7538	#endif
7539
7540	/*
7541	* Load the VCPU's VMCS as the current (and active) one.
7542	*/
7543	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7544	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7545	if (RT_FAILURE(rc))
7546	return rc;
7547
7548	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7549	pVCpu->hm.s.fLeaveDone = false;
7550	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7551
7552	return VINF_SUCCESS;
7553	}
7554
7555
7556	/**
7557	* The thread-context callback (only on platforms which support it).
7558	*
7559	* @param enmEvent The thread-context event.
7560	* @param pVCpu Pointer to the VMCPU.
7561	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7562	* @thread EMT(pVCpu)
7563	*/
7564	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7565	{
7566	NOREF(fGlobalInit);
7567
7568	switch (enmEvent)
7569	{
7570	case RTTHREADCTXEVENT_PREEMPTING:
7571	{
7572	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7573	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7574	VMCPU_ASSERT_EMT(pVCpu);
7575
7576	PVM pVM = pVCpu->CTX_SUFF(pVM);
7577	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7578
7579	/* No longjmps (logger flushes, locks) in this fragile context. */
7580	VMMRZCallRing3Disable(pVCpu);
7581	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7582
7583	/*
7584	* Restore host-state (FPU, debug etc.)
7585	*/
7586	if (!pVCpu->hm.s.fLeaveDone)
7587	{
7588	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7589	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7590	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7591	pVCpu->hm.s.fLeaveDone = true;
7592	}
7593
7594	/* Leave HM context, takes care of local init (term). */
7595	int rc = HMR0LeaveCpu(pVCpu);
7596	AssertRC(rc); NOREF(rc);
7597
7598	/* Restore longjmp state. */
7599	VMMRZCallRing3Enable(pVCpu);
7600	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7601	break;
7602	}
7603
7604	case RTTHREADCTXEVENT_RESUMED:
7605	{
7606	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7607	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7608	VMCPU_ASSERT_EMT(pVCpu);
7609
7610	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7611	VMMRZCallRing3Disable(pVCpu);
7612	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7613
7614	/* Initialize the bare minimum state required for HM. This takes care of
7615	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7616	int rc = HMR0EnterCpu(pVCpu);
7617	AssertRC(rc);
7618	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7619
7620	/* Load the active VMCS as the current one. */
7621	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7622	{
7623	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7624	AssertRC(rc); NOREF(rc);
7625	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7626	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7627	}
7628	pVCpu->hm.s.fLeaveDone = false;
7629
7630	/* Restore longjmp state. */
7631	VMMRZCallRing3Enable(pVCpu);
7632	break;
7633	}
7634
7635	default:
7636	break;
7637	}
7638	}
7639
7640
7641	/**
7642	* Saves the host state in the VMCS host-state.
7643	* Sets up the VM-exit MSR-load area.
7644	*
7645	* The CPU state will be loaded from these fields on every successful VM-exit.
7646	*
7647	* @returns VBox status code.
7648	* @param pVM Pointer to the VM.
7649	* @param pVCpu Pointer to the VMCPU.
7650	*
7651	* @remarks No-long-jump zone!!!
7652	*/
7653	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7654	{
7655	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7656
7657	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7658	return VINF_SUCCESS;
7659
7660	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7661	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7662
7663	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7664	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7665
7666	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7667	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7668
7669	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7670	return rc;
7671	}
7672
7673
7674	/**
7675	* Saves the host state in the VMCS host-state.
7676	*
7677	* @returns VBox status code.
7678	* @param pVM Pointer to the VM.
7679	* @param pVCpu Pointer to the VMCPU.
7680	*
7681	* @remarks No-long-jump zone!!!
7682	*/
7683	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7684	{
7685	AssertPtr(pVM);
7686	AssertPtr(pVCpu);
7687
7688	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7689
7690	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7691	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7692	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7693	return hmR0VmxSaveHostState(pVM, pVCpu);
7694	}
7695
7696
7697	/**
7698	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7699	* loaded from these fields on every successful VM-entry.
7700	*
7701	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7702	* Sets up the VM-entry controls.
7703	* Sets up the appropriate VMX non-root function to execute guest code based on
7704	* the guest CPU mode.
7705	*
7706	* @returns VBox status code.
7707	* @param pVM Pointer to the VM.
7708	* @param pVCpu Pointer to the VMCPU.
7709	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7710	* out-of-sync. Make sure to update the required fields
7711	* before using them.
7712	*
7713	* @remarks No-long-jump zone!!!
7714	*/
7715	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7716	{
7717	AssertPtr(pVM);
7718	AssertPtr(pVCpu);
7719	AssertPtr(pMixedCtx);
7720	HMVMX_ASSERT_PREEMPT_SAFE();
7721
7722	#ifdef LOG_ENABLED
7723	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7724	* probably not initialized yet? Anyway this will do for now.
7725	*
7726	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7727	* interface and disable ring-3 calls when thread-context hooks are not
7728	* available. */
7729	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7730	VMMR0LogFlushDisable(pVCpu);
7731	#endif
7732
7733	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7734
7735	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7736
7737	/* Determine real-on-v86 mode. */
7738	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7739	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7740	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7741	{
7742	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7743	}
7744
7745	/*
7746	* Load the guest-state into the VMCS.
7747	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7748	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7749	*/
7750	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7751	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7752
7753	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7754	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7755	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7756
7757	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7758	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7759	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7760
7761	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7762	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7763
7764	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7765	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7766
7767	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7768	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7769	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7770
7771	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7772	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7773
7774	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7775	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7776
7777	/*
7778	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7779	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7780	*/
7781	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7782	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7783
7784	/* Clear any unused and reserved bits. */
7785	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7786
7787	#ifdef LOG_ENABLED
7788	/* Only reenable log-flushing if the caller has it enabled. */
7789	if (!fCallerDisabledLogFlush)
7790	VMMR0LogFlushEnable(pVCpu);
7791	#endif
7792
7793	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7794	return rc;
7795	}
7796
7797
7798	/**
7799	* Loads the state shared between the host and guest into the VMCS.
7800	*
7801	* @param pVM Pointer to the VM.
7802	* @param pVCpu Pointer to the VMCPU.
7803	* @param pCtx Pointer to the guest-CPU context.
7804	*
7805	* @remarks No-long-jump zone!!!
7806	*/
7807	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7808	{
7809	NOREF(pVM);
7810
7811	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7812	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7813
7814	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7815	{
7816	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7817	AssertRC(rc);
7818	}
7819
7820	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7821	{
7822	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7823	AssertRC(rc);
7824
7825	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7826	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7827	{
7828	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7829	AssertRC(rc);
7830	}
7831	}
7832
7833	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7834	{
7835	#if HC_ARCH_BITS == 64
7836	if (pVM->hm.s.fAllow64BitGuests)
7837	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7838	#endif
7839	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7840	}
7841
7842	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7843	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7844	}
7845
7846
7847	/**
7848	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7849	*
7850	* @param pVM Pointer to the VM.
7851	* @param pVCpu Pointer to the VMCPU.
7852	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7853	* out-of-sync. Make sure to update the required fields
7854	* before using them.
7855	*/
7856	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7857	{
7858	HMVMX_ASSERT_PREEMPT_SAFE();
7859
7860	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7861	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7862	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7863	#endif
7864
7865	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7866	{
7867	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7868	AssertRC(rc);
7869	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7870	}
7871	else if (HMCPU_CF_VALUE(pVCpu))
7872	{
7873	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7874	AssertRC(rc);
7875	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7876	}
7877
7878	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7879	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7880	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7881	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7882
7883	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7884	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7885	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7886	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7887	#endif
7888	}
7889
7890
7891	/**
7892	* Does the preparations before executing guest code in VT-x.
7893	*
7894	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7895	* recompiler. We must be cautious what we do here regarding committing
7896	* guest-state information into the VMCS assuming we assuredly execute the
7897	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7898	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7899	* so that the recompiler can (and should) use them when it resumes guest
7900	* execution. Otherwise such operations must be done when we can no longer
7901	* exit to ring-3.
7902	*
7903	* @returns Strict VBox status code.
7904	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7905	* have been disabled.
7906	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7907	* double-fault into the guest.
7908	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7909	*
7910	* @param pVM Pointer to the VM.
7911	* @param pVCpu Pointer to the VMCPU.
7912	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7913	* out-of-sync. Make sure to update the required fields
7914	* before using them.
7915	* @param pVmxTransient Pointer to the VMX transient structure.
7916	*/
7917	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7918	{
7919	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7920
7921	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7922	PGMRZDynMapFlushAutoSet(pVCpu);
7923	#endif
7924
7925	/* Check force flag actions that might require us to go back to ring-3. */
7926	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7927	if (rc != VINF_SUCCESS)
7928	return rc;
7929
7930	#ifndef IEM_VERIFICATION_MODE_FULL
7931	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7932	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7933	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7934	{
7935	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7936	RTGCPHYS GCPhysApicBase;
7937	GCPhysApicBase = pMixedCtx->msrApicBase;
7938	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7939
7940	/* Unalias any existing mapping. */
7941	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7942	AssertRCReturn(rc, rc);
7943
7944	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7945	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7946	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7947	AssertRCReturn(rc, rc);
7948
7949	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7950	}
7951	#endif /* !IEM_VERIFICATION_MODE_FULL */
7952
7953	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7954	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7955
7956	/*
7957	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7958	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7959	*/
7960	if (TRPMHasTrap(pVCpu))
7961	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7962	else if (!pVCpu->hm.s.Event.fPending)
7963	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7964
7965	/*
7966	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7967	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7968	*/
7969	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7970	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7971	{
7972	Assert(rc == VINF_EM_RESET);
7973	return rc;
7974	}
7975
7976	/*
7977	* No longjmps to ring-3 from this point on!!!
7978	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7979	* This also disables flushing of the R0-logger instance (if any).
7980	*/
7981	VMMRZCallRing3Disable(pVCpu);
7982
7983	/*
7984	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7985	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7986	*
7987	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7988	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7989	*
7990	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7991	* executing guest code.
7992	*/
7993	pVmxTransient->uEflags = ASMIntDisableFlags();
7994	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7995	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7996	{
7997	hmR0VmxClearEventVmcs(pVCpu);
7998	ASMSetFlags(pVmxTransient->uEflags);
7999	VMMRZCallRing3Enable(pVCpu);
8000	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8001	return VINF_EM_RAW_TO_R3;
8002	}
8003	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8004	{
8005	hmR0VmxClearEventVmcs(pVCpu);
8006	ASMSetFlags(pVmxTransient->uEflags);
8007	VMMRZCallRing3Enable(pVCpu);
8008	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8009	return VINF_EM_RAW_INTERRUPT;
8010	}
8011
8012	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8013	pVCpu->hm.s.Event.fPending = false;
8014
8015	return VINF_SUCCESS;
8016	}
8017
8018
8019	/**
8020	* Prepares to run guest code in VT-x and we've committed to doing so. This
8021	* means there is no backing out to ring-3 or anywhere else at this
8022	* point.
8023	*
8024	* @param pVM Pointer to the VM.
8025	* @param pVCpu Pointer to the VMCPU.
8026	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8027	* out-of-sync. Make sure to update the required fields
8028	* before using them.
8029	* @param pVmxTransient Pointer to the VMX transient structure.
8030	*
8031	* @remarks Called with preemption disabled.
8032	* @remarks No-long-jump zone!!!
8033	*/
8034	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8035	{
8036	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8037	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8038	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8039
8040	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8041	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8042
8043	/*
8044	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8045	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8046	* Reload only the necessary state, the assertion will catch if other parts of the code
8047	* change.
8048	*/
8049	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8050	{
8051	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8052	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8053	}
8054
8055	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8056	if (!CPUMIsGuestFPUStateActive(pVCpu))
8057	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8058	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8059	#endif
8060
8061	if ( pVCpu->hm.s.fUseGuestFpu
8062	&& !CPUMIsGuestFPUStateActive(pVCpu))
8063	{
8064	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8065	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8066	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8067	}
8068
8069	/*
8070	* The host MSR values the very first time around won't be updated, so we need to
8071	* fill those values in. Subsequently, it's updated as part of the host state.
8072	*/
8073	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8074	&& pVCpu->hm.s.vmx.cMsrs > 0)
8075	{
8076	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8077	}
8078
8079	/*
8080	* Load the host state bits as we may've been preempted (only happens when
8081	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8082	*/
8083	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8084	{
8085	/* This ASSUMES that pfnStartVM has been set up already. */
8086	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8087	AssertRC(rc);
8088	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8089	}
8090	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8091
8092	/*
8093	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8094	*/
8095	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8096	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8097	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8098
8099	/* Store status of the shared guest-host state at the time of VM-entry. */
8100	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8101	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8102	{
8103	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8104	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8105	}
8106	else
8107	#endif
8108	{
8109	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8110	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8111	}
8112	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8113
8114	/*
8115	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8116	*/
8117	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8118	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8119
8120	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8121	RTCPUID idCurrentCpu = pCpu->idCpu;
8122	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8123	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8124	{
8125	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8126	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8127	}
8128
8129	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8130	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8131	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8132	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8133
8134	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8135
8136	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8137	to start executing. */
8138
8139	/*
8140	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8141	*/
8142	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8143	{
8144	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8145	{
8146	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8147	AssertRC(rc2);
8148	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8149	uint64_t u64GuestTscAuxMsr;
8150	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTscAuxMsr);
8151	AssertRC(rc2);
8152	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, u64GuestTscAuxMsr, true /* fUpdateHostMsr */);
8153	}
8154	else
8155	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8156	}
8157	#ifdef VBOX_STRICT
8158	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8159	#endif
8160	}
8161
8162
8163	/**
8164	* Performs some essential restoration of state after running guest code in
8165	* VT-x.
8166	*
8167	* @param pVM Pointer to the VM.
8168	* @param pVCpu Pointer to the VMCPU.
8169	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8170	* out-of-sync. Make sure to update the required fields
8171	* before using them.
8172	* @param pVmxTransient Pointer to the VMX transient structure.
8173	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8174	*
8175	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8176	*
8177	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8178	* unconditionally when it is safe to do so.
8179	*/
8180	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8181	{
8182	NOREF(pVM);
8183
8184	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8185
8186	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8187	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8188	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8189	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8190	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8191
8192	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8193	{
8194	/** @todo Find a way to fix hardcoding a guestimate. */
8195	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8196	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8197	}
8198
8199	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8200	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8201	Assert(!(ASMGetFlags() & X86_EFL_IF));
8202	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8203
8204	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8205	if (CPUMIsGuestFPUStateActive(pVCpu))
8206	{
8207	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8208	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8209	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8210	}
8211	#endif
8212
8213	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8214	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8215	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8216	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8217
8218	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8219	uint32_t uExitReason;
8220	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8221	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8222	AssertRC(rc);
8223	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8224	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8225
8226	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8227	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8228	{
8229	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8230	pVmxTransient->fVMEntryFailed));
8231	return;
8232	}
8233
8234	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8235	{
8236	/* Update the guest interruptibility-state from the VMCS. */
8237	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8238	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8239	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8240	AssertRC(rc);
8241	#endif
8242	/*
8243	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8244	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8245	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8246	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8247	*/
8248	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8249	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8250	{
8251	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8252	AssertRC(rc);
8253	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8254	}
8255	}
8256	}
8257
8258
8259
8260	/**
8261	* Runs the guest code using VT-x the normal way.
8262	*
8263	* @returns VBox status code.
8264	* @param pVM Pointer to the VM.
8265	* @param pVCpu Pointer to the VMCPU.
8266	* @param pCtx Pointer to the guest-CPU context.
8267	*
8268	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8269	*/
8270	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8271	{
8272	VMXTRANSIENT VmxTransient;
8273	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8274	int rc = VERR_INTERNAL_ERROR_5;
8275	uint32_t cLoops = 0;
8276
8277	for (;; cLoops++)
8278	{
8279	Assert(!HMR0SuspendPending());
8280	HMVMX_ASSERT_CPU_SAFE();
8281
8282	/* Preparatory work for running guest code, this may force us to return
8283	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8284	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8285	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8286	if (rc != VINF_SUCCESS)
8287	break;
8288
8289	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8290	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8291	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8292
8293	/* Restore any residual host-state and save any bits shared between host
8294	and guest into the guest-CPU state. Re-enables interrupts! */
8295	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8296
8297	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8298	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8299	{
8300	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8301	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8302	return rc;
8303	}
8304
8305	/* Handle the VM-exit. */
8306	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8308	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8309	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8310	HMVMX_START_EXIT_DISPATCH_PROF();
8311	#ifdef HMVMX_USE_FUNCTION_TABLE
8312	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8313	#else
8314	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8315	#endif
8316	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8317	if (rc != VINF_SUCCESS)
8318	break;
8319	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8320	{
8321	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8322	rc = VINF_EM_RAW_INTERRUPT;
8323	break;
8324	}
8325	}
8326
8327	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8328	return rc;
8329	}
8330
8331
8332	/**
8333	* Single steps guest code using VT-x.
8334	*
8335	* @returns VBox status code.
8336	* @param pVM Pointer to the VM.
8337	* @param pVCpu Pointer to the VMCPU.
8338	* @param pCtx Pointer to the guest-CPU context.
8339	*
8340	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8341	*/
8342	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8343	{
8344	VMXTRANSIENT VmxTransient;
8345	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8346	int rc = VERR_INTERNAL_ERROR_5;
8347	uint32_t cLoops = 0;
8348	uint16_t uCsStart = pCtx->cs.Sel;
8349	uint64_t uRipStart = pCtx->rip;
8350
8351	for (;; cLoops++)
8352	{
8353	Assert(!HMR0SuspendPending());
8354	HMVMX_ASSERT_CPU_SAFE();
8355
8356	/* Preparatory work for running guest code, this may force us to return
8357	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8358	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8359	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8360	if (rc != VINF_SUCCESS)
8361	break;
8362
8363	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8364	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8365	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8366
8367	/* Restore any residual host-state and save any bits shared between host
8368	and guest into the guest-CPU state. Re-enables interrupts! */
8369	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8370
8371	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8372	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8373	{
8374	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8375	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8376	return rc;
8377	}
8378
8379	/* Handle the VM-exit. */
8380	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8382	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8383	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8384	HMVMX_START_EXIT_DISPATCH_PROF();
8385	#ifdef HMVMX_USE_FUNCTION_TABLE
8386	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8387	#else
8388	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8389	#endif
8390	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8391	if (rc != VINF_SUCCESS)
8392	break;
8393	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8394	{
8395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8396	rc = VINF_EM_RAW_INTERRUPT;
8397	break;
8398	}
8399
8400	/*
8401	* Did the RIP change, if so, consider it a single step.
8402	* Otherwise, make sure one of the TFs gets set.
8403	*/
8404	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8405	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8406	AssertRCReturn(rc2, rc2);
8407	if ( pCtx->rip != uRipStart
8408	\|\| pCtx->cs.Sel != uCsStart)
8409	{
8410	rc = VINF_EM_DBG_STEPPED;
8411	break;
8412	}
8413	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8414	}
8415
8416	/*
8417	* Clear the X86_EFL_TF if necessary.
8418	*/
8419	if (pVCpu->hm.s.fClearTrapFlag)
8420	{
8421	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8422	AssertRCReturn(rc2, rc2);
8423	pVCpu->hm.s.fClearTrapFlag = false;
8424	pCtx->eflags.Bits.u1TF = 0;
8425	}
8426	/** @todo there seems to be issues with the resume flag when the monitor trap
8427	* flag is pending without being used. Seen early in bios init when
8428	* accessing APIC page in prot mode. */
8429
8430	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8431	return rc;
8432	}
8433
8434
8435	/**
8436	* Runs the guest code using VT-x.
8437	*
8438	* @returns VBox status code.
8439	* @param pVM Pointer to the VM.
8440	* @param pVCpu Pointer to the VMCPU.
8441	* @param pCtx Pointer to the guest-CPU context.
8442	*/
8443	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8444	{
8445	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8446	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8447	HMVMX_ASSERT_PREEMPT_SAFE();
8448
8449	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8450
8451	int rc;
8452	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8453	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8454	else
8455	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8456
8457	if (rc == VERR_EM_INTERPRETER)
8458	rc = VINF_EM_RAW_EMULATE_INSTR;
8459	else if (rc == VINF_EM_RESET)
8460	rc = VINF_EM_TRIPLE_FAULT;
8461
8462	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8463	if (RT_FAILURE(rc2))
8464	{
8465	pVCpu->hm.s.u32HMError = rc;
8466	rc = rc2;
8467	}
8468	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8469	return rc;
8470	}
8471
8472
8473	#ifndef HMVMX_USE_FUNCTION_TABLE
8474	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8475	{
8476	#ifdef DEBUG_ramshankar
8477	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8478	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8479	#endif
8480	int rc;
8481	switch (rcReason)
8482	{
8483	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8508	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8509	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8510	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8511	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8512	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8513	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8514	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8515	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8516
8517	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8518	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8519	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8520	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8521	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8522	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8523	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8524	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8525	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8526
8527	case VMX_EXIT_VMCALL:
8528	case VMX_EXIT_VMCLEAR:
8529	case VMX_EXIT_VMLAUNCH:
8530	case VMX_EXIT_VMPTRLD:
8531	case VMX_EXIT_VMPTRST:
8532	case VMX_EXIT_VMREAD:
8533	case VMX_EXIT_VMRESUME:
8534	case VMX_EXIT_VMWRITE:
8535	case VMX_EXIT_VMXOFF:
8536	case VMX_EXIT_VMXON:
8537	case VMX_EXIT_INVEPT:
8538	case VMX_EXIT_INVVPID:
8539	case VMX_EXIT_VMFUNC:
8540	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8541	break;
8542	default:
8543	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8544	break;
8545	}
8546	return rc;
8547	}
8548	#endif
8549
8550	#ifdef DEBUG
8551	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8552	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8553	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8554
8555	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8556	do \
8557	{ \
8558	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8559	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8560	} while (0)
8561
8562	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8563	do { \
8564	AssertPtr(pVCpu); \
8565	AssertPtr(pMixedCtx); \
8566	AssertPtr(pVmxTransient); \
8567	Assert(pVmxTransient->fVMEntryFailed == false); \
8568	Assert(ASMIntAreEnabled()); \
8569	HMVMX_ASSERT_PREEMPT_SAFE(); \
8570	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8571	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8572	HMVMX_ASSERT_PREEMPT_SAFE(); \
8573	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8574	HMVMX_ASSERT_PREEMPT_CPUID(); \
8575	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8576	} while (0)
8577
8578	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8579	do { \
8580	Log4Func(("\n")); \
8581	} while (0)
8582	#else /* Release builds */
8583	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8584	do { \
8585	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8586	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8587	} while (0)
8588	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8589	#endif
8590
8591
8592	/**
8593	* Advances the guest RIP after reading it from the VMCS.
8594	*
8595	* @returns VBox status code.
8596	* @param pVCpu Pointer to the VMCPU.
8597	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8598	* out-of-sync. Make sure to update the required fields
8599	* before using them.
8600	* @param pVmxTransient Pointer to the VMX transient structure.
8601	*
8602	* @remarks No-long-jump zone!!!
8603	*/
8604	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8605	{
8606	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8607	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8608	AssertRCReturn(rc, rc);
8609
8610	pMixedCtx->rip += pVmxTransient->cbInstr;
8611	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8612	return rc;
8613	}
8614
8615
8616	/**
8617	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8618	* and update error record fields accordingly.
8619	*
8620	* @return VMX_IGS_* return codes.
8621	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8622	* wrong with the guest state.
8623	*
8624	* @param pVM Pointer to the VM.
8625	* @param pVCpu Pointer to the VMCPU.
8626	* @param pCtx Pointer to the guest-CPU state.
8627	*/
8628	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8629	{
8630	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8631	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8632	uError = (err); \
8633	break; \
8634	} else do { } while (0)
8635
8636	int rc;
8637	uint32_t uError = VMX_IGS_ERROR;
8638	uint32_t u32Val;
8639	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8640
8641	do
8642	{
8643	/*
8644	* CR0.
8645	*/
8646	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8647	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8648	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8649	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8650	if (fUnrestrictedGuest)
8651	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8652
8653	uint32_t u32GuestCR0;
8654	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8655	AssertRCBreak(rc);
8656	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8657	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8658	if ( !fUnrestrictedGuest
8659	&& (u32GuestCR0 & X86_CR0_PG)
8660	&& !(u32GuestCR0 & X86_CR0_PE))
8661	{
8662	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8663	}
8664
8665	/*
8666	* CR4.
8667	*/
8668	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8669	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8670
8671	uint32_t u32GuestCR4;
8672	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8673	AssertRCBreak(rc);
8674	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8675	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8676
8677	/*
8678	* IA32_DEBUGCTL MSR.
8679	*/
8680	uint64_t u64Val;
8681	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8682	AssertRCBreak(rc);
8683	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8684	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8685	{
8686	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8687	}
8688	uint64_t u64DebugCtlMsr = u64Val;
8689
8690	#ifdef VBOX_STRICT
8691	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8692	AssertRCBreak(rc);
8693	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8694	#endif
8695	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8696
8697	/*
8698	* RIP and RFLAGS.
8699	*/
8700	uint32_t u32Eflags;
8701	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8702	if (HMVMX_IS_64BIT_HOST_MODE())
8703	{
8704	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8705	AssertRCBreak(rc);
8706	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8707	if ( !fLongModeGuest
8708	\|\| !pCtx->cs.Attr.n.u1Long)
8709	{
8710	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8711	}
8712	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8713	* must be identical if the "IA32e mode guest" VM-entry control is 1
8714	* and CS.L is 1. No check applies if the CPU supports 64
8715	* linear-address bits. */
8716
8717	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8718	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8719	AssertRCBreak(rc);
8720	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8721	VMX_IGS_RFLAGS_RESERVED);
8722	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8723	u32Eflags = u64Val;
8724	}
8725	else
8726	#endif
8727	{
8728	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8729	AssertRCBreak(rc);
8730	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8731	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8732	}
8733
8734	if ( fLongModeGuest
8735	\|\| ( fUnrestrictedGuest
8736	&& !(u32GuestCR0 & X86_CR0_PE)))
8737	{
8738	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8739	}
8740
8741	uint32_t u32EntryInfo;
8742	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8743	AssertRCBreak(rc);
8744	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8745	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8746	{
8747	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8748	}
8749
8750	/*
8751	* 64-bit checks.
8752	*/
8753	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8754	if (HMVMX_IS_64BIT_HOST_MODE())
8755	{
8756	if ( fLongModeGuest
8757	&& !fUnrestrictedGuest)
8758	{
8759	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8760	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8761	}
8762
8763	if ( !fLongModeGuest
8764	&& (u32GuestCR4 & X86_CR4_PCIDE))
8765	{
8766	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8767	}
8768
8769	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8770	* 51:32 beyond the processor's physical-address width are 0. */
8771
8772	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8773	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8774	{
8775	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8776	}
8777
8778	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8779	AssertRCBreak(rc);
8780	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8781
8782	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8783	AssertRCBreak(rc);
8784	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8785	}
8786	#endif
8787
8788	/*
8789	* PERF_GLOBAL MSR.
8790	*/
8791	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8792	{
8793	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8794	AssertRCBreak(rc);
8795	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8796	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8797	}
8798
8799	/*
8800	* PAT MSR.
8801	*/
8802	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8803	{
8804	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8805	AssertRCBreak(rc);
8806	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8807	for (unsigned i = 0; i < 8; i++)
8808	{
8809	uint8_t u8Val = (u64Val & 0x7);
8810	if ( u8Val != 0 /* UC */
8811	\|\| u8Val != 1 /* WC */
8812	\|\| u8Val != 4 /* WT */
8813	\|\| u8Val != 5 /* WP */
8814	\|\| u8Val != 6 /* WB */
8815	\|\| u8Val != 7 /* UC- */)
8816	{
8817	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8818	}
8819	u64Val >>= 3;
8820	}
8821	}
8822
8823	/*
8824	* EFER MSR.
8825	*/
8826	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8827	{
8828	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8829	AssertRCBreak(rc);
8830	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8831	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8832	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8833	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8834	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8835	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8836	}
8837
8838	/*
8839	* Segment registers.
8840	*/
8841	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8842	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8843	if (!(u32Eflags & X86_EFL_VM))
8844	{
8845	/* CS */
8846	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8847	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8848	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8849	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8850	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8851	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8852	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8853	/* CS cannot be loaded with NULL in protected mode. */
8854	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8855	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8856	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8857	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8858	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8859	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8860	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8861	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8862	else
8863	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8864
8865	/* SS */
8866	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8867	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8868	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8869	if ( !(pCtx->cr0 & X86_CR0_PE)
8870	\|\| pCtx->cs.Attr.n.u4Type == 3)
8871	{
8872	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8873	}
8874	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8875	{
8876	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8877	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8878	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8879	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8880	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8881	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8882	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8883	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8884	}
8885
8886	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8887	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8888	{
8889	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8890	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8891	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8892	\|\| pCtx->ds.Attr.n.u4Type > 11
8893	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8894	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8895	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8896	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8897	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8898	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8899	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8900	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8901	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8902	}
8903	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8904	{
8905	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8906	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8907	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8908	\|\| pCtx->es.Attr.n.u4Type > 11
8909	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8910	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8911	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8912	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8913	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8914	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8915	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8916	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8917	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8918	}
8919	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8920	{
8921	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8922	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8923	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8924	\|\| pCtx->fs.Attr.n.u4Type > 11
8925	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8926	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8927	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8928	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8929	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8930	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8931	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8932	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8933	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8934	}
8935	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8936	{
8937	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8938	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8939	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8940	\|\| pCtx->gs.Attr.n.u4Type > 11
8941	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8942	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8943	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8944	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8945	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8946	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8947	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8948	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8949	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8950	}
8951	/* 64-bit capable CPUs. */
8952	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8953	if (HMVMX_IS_64BIT_HOST_MODE())
8954	{
8955	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8956	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8957	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8958	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8959	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8960	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8961	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8962	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8963	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8964	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8965	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8966	}
8967	#endif
8968	}
8969	else
8970	{
8971	/* V86 mode checks. */
8972	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8973	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8974	{
8975	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8976	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8977	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8978	}
8979	else
8980	{
8981	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8982	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8983	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8984	}
8985
8986	/* CS */
8987	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8988	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8989	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8990	/* SS */
8991	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8992	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8993	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8994	/* DS */
8995	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8996	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8997	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8998	/* ES */
8999	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9000	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9001	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9002	/* FS */
9003	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9004	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9005	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9006	/* GS */
9007	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9008	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9009	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9010	/* 64-bit capable CPUs. */
9011	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9012	if (HMVMX_IS_64BIT_HOST_MODE())
9013	{
9014	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9015	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9016	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9017	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9018	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9019	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9020	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9021	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9022	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9023	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9024	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9025	}
9026	#endif
9027	}
9028
9029	/*
9030	* TR.
9031	*/
9032	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9033	/* 64-bit capable CPUs. */
9034	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9035	if (HMVMX_IS_64BIT_HOST_MODE())
9036	{
9037	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9038	}
9039	#endif
9040	if (fLongModeGuest)
9041	{
9042	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9043	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9044	}
9045	else
9046	{
9047	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9048	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9049	VMX_IGS_TR_ATTR_TYPE_INVALID);
9050	}
9051	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9052	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9053	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9054	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9055	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9056	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9057	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9058	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9059
9060	/*
9061	* GDTR and IDTR.
9062	*/
9063	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9064	if (HMVMX_IS_64BIT_HOST_MODE())
9065	{
9066	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9067	AssertRCBreak(rc);
9068	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9069
9070	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9071	AssertRCBreak(rc);
9072	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9073	}
9074	#endif
9075
9076	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9077	AssertRCBreak(rc);
9078	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9079
9080	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9081	AssertRCBreak(rc);
9082	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9083
9084	/*
9085	* Guest Non-Register State.
9086	*/
9087	/* Activity State. */
9088	uint32_t u32ActivityState;
9089	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9090	AssertRCBreak(rc);
9091	HMVMX_CHECK_BREAK( !u32ActivityState
9092	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9093	VMX_IGS_ACTIVITY_STATE_INVALID);
9094	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9095	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9096	uint32_t u32IntrState;
9097	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9098	AssertRCBreak(rc);
9099	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9100	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9101	{
9102	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9103	}
9104
9105	/** @todo Activity state and injecting interrupts. Left as a todo since we
9106	* currently don't use activity states but ACTIVE. */
9107
9108	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9109	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9110
9111	/* Guest interruptibility-state. */
9112	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9113	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9114	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9115	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9116	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9117	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9118	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9119	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9120	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9121	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9122	{
9123	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9124	{
9125	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9126	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9127	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9128	}
9129	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9130	{
9131	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9132	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9133	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9134	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9135	}
9136	}
9137	/** @todo Assumes the processor is not in SMM. */
9138	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9139	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9140	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9141	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9142	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9143	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9144	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9145	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9146	{
9147	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9148	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9149	}
9150
9151	/* Pending debug exceptions. */
9152	if (HMVMX_IS_64BIT_HOST_MODE())
9153	{
9154	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9155	AssertRCBreak(rc);
9156	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9157	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9158	u32Val = u64Val; /* For pending debug exceptions checks below. */
9159	}
9160	else
9161	{
9162	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9163	AssertRCBreak(rc);
9164	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9165	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9166	}
9167
9168	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9169	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9170	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9171	{
9172	if ( (u32Eflags & X86_EFL_TF)
9173	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9174	{
9175	/* Bit 14 is PendingDebug.BS. */
9176	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9177	}
9178	if ( !(u32Eflags & X86_EFL_TF)
9179	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9180	{
9181	/* Bit 14 is PendingDebug.BS. */
9182	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9183	}
9184	}
9185
9186	/* VMCS link pointer. */
9187	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9188	AssertRCBreak(rc);
9189	if (u64Val != UINT64_C(0xffffffffffffffff))
9190	{
9191	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9192	/** @todo Bits beyond the processor's physical-address width MBZ. */
9193	/** @todo 32-bit located in memory referenced by value of this field (as a
9194	* physical address) must contain the processor's VMCS revision ID. */
9195	/** @todo SMM checks. */
9196	}
9197
9198	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9199	* not using Nested Paging? */
9200	if ( pVM->hm.s.fNestedPaging
9201	&& !fLongModeGuest
9202	&& CPUMIsGuestInPAEModeEx(pCtx))
9203	{
9204	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9205	AssertRCBreak(rc);
9206	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9207
9208	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9209	AssertRCBreak(rc);
9210	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9211
9212	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9213	AssertRCBreak(rc);
9214	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9215
9216	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9217	AssertRCBreak(rc);
9218	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9219	}
9220
9221	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9222	if (uError == VMX_IGS_ERROR)
9223	uError = VMX_IGS_REASON_NOT_FOUND;
9224	} while (0);
9225
9226	pVCpu->hm.s.u32HMError = uError;
9227	return uError;
9228
9229	#undef HMVMX_ERROR_BREAK
9230	#undef HMVMX_CHECK_BREAK
9231	}
9232
9233	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9234	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9235	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9236
9237	/** @name VM-exit handlers.
9238	* @{
9239	*/
9240
9241	/**
9242	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9243	*/
9244	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9245	{
9246	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9247	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9248	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9249	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9250	return VINF_SUCCESS;
9251	return VINF_EM_RAW_INTERRUPT;
9252	}
9253
9254
9255	/**
9256	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9257	*/
9258	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9259	{
9260	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9261	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9262
9263	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9264	AssertRCReturn(rc, rc);
9265
9266	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9267	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9268	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9269	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9270
9271	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9272	{
9273	/*
9274	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9275	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9276	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9277	*
9278	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9279	*/
9280	VMXDispatchHostNmi();
9281	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9282	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9283	return VINF_SUCCESS;
9284	}
9285
9286	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9287	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9288	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9289	{
9290	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9291	return VINF_SUCCESS;
9292	}
9293	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9294	{
9295	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9296	return rc;
9297	}
9298
9299	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9300	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9301	switch (uIntType)
9302	{
9303	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9304	Assert(uVector == X86_XCPT_DB);
9305	/* no break */
9306	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9307	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9308	/* no break */
9309	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9310	{
9311	switch (uVector)
9312	{
9313	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9314	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9315	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9316	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9317	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9318	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9319	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9320	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9321	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9322	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9323	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9324	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9325	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9326	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9327	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9328	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9329	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9330	#endif
9331	default:
9332	{
9333	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9334	AssertRCReturn(rc, rc);
9335
9336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9337	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9338	{
9339	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9340	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9341	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9342
9343	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9344	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9345	AssertRCReturn(rc, rc);
9346	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9347	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9348	0 /* GCPtrFaultAddress */);
9349	AssertRCReturn(rc, rc);
9350	}
9351	else
9352	{
9353	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9354	pVCpu->hm.s.u32HMError = uVector;
9355	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9356	}
9357	break;
9358	}
9359	}
9360	break;
9361	}
9362
9363	default:
9364	{
9365	pVCpu->hm.s.u32HMError = uExitIntInfo;
9366	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9367	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9368	break;
9369	}
9370	}
9371	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9372	return rc;
9373	}
9374
9375
9376	/**
9377	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9378	*/
9379	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9380	{
9381	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9382
9383	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9384	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9385	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9386	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9387	AssertRCReturn(rc, rc);
9388
9389	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9391	return VINF_SUCCESS;
9392	}
9393
9394
9395	/**
9396	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9397	*/
9398	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9399	{
9400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9401	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9402	HMVMX_RETURN_UNEXPECTED_EXIT();
9403	}
9404
9405
9406	/**
9407	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9408	*/
9409	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9410	{
9411	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9413	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9414	}
9415
9416
9417	/**
9418	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9419	*/
9420	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9421	{
9422	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9423	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9424	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9425	}
9426
9427
9428	/**
9429	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9430	*/
9431	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9432	{
9433	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9434	PVM pVM = pVCpu->CTX_SUFF(pVM);
9435	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9436	if (RT_LIKELY(rc == VINF_SUCCESS))
9437	{
9438	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9439	Assert(pVmxTransient->cbInstr == 2);
9440	}
9441	else
9442	{
9443	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9444	rc = VERR_EM_INTERPRETER;
9445	}
9446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9447	return rc;
9448	}
9449
9450
9451	/**
9452	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9453	*/
9454	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9455	{
9456	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9457	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9458	AssertRCReturn(rc, rc);
9459
9460	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9461	return VINF_EM_RAW_EMULATE_INSTR;
9462
9463	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9464	HMVMX_RETURN_UNEXPECTED_EXIT();
9465	}
9466
9467
9468	/**
9469	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9470	*/
9471	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9472	{
9473	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9474	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9475	AssertRCReturn(rc, rc);
9476
9477	PVM pVM = pVCpu->CTX_SUFF(pVM);
9478	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9479	if (RT_LIKELY(rc == VINF_SUCCESS))
9480	{
9481	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9482	Assert(pVmxTransient->cbInstr == 2);
9483	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9484	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9485	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9486	}
9487	else
9488	{
9489	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9490	rc = VERR_EM_INTERPRETER;
9491	}
9492	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9493	return rc;
9494	}
9495
9496
9497	/**
9498	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9499	*/
9500	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9501	{
9502	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9503	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9504	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9505	AssertRCReturn(rc, rc);
9506
9507	PVM pVM = pVCpu->CTX_SUFF(pVM);
9508	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9509	if (RT_LIKELY(rc == VINF_SUCCESS))
9510	{
9511	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9512	Assert(pVmxTransient->cbInstr == 3);
9513	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9514	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9515	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9516	}
9517	else
9518	{
9519	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9520	rc = VERR_EM_INTERPRETER;
9521	}
9522	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9523	return rc;
9524	}
9525
9526
9527	/**
9528	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9529	*/
9530	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9531	{
9532	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9533	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9534	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9535	AssertRCReturn(rc, rc);
9536
9537	PVM pVM = pVCpu->CTX_SUFF(pVM);
9538	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9539	if (RT_LIKELY(rc == VINF_SUCCESS))
9540	{
9541	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9542	Assert(pVmxTransient->cbInstr == 2);
9543	}
9544	else
9545	{
9546	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9547	rc = VERR_EM_INTERPRETER;
9548	}
9549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9550	return rc;
9551	}
9552
9553
9554	/**
9555	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9556	*/
9557	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9558	{
9559	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9560	PVM pVM = pVCpu->CTX_SUFF(pVM);
9561	Assert(!pVM->hm.s.fNestedPaging);
9562
9563	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9564	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9565	AssertRCReturn(rc, rc);
9566
9567	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9568	rc = VBOXSTRICTRC_VAL(rc2);
9569	if (RT_LIKELY(rc == VINF_SUCCESS))
9570	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9571	else
9572	{
9573	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9574	pVmxTransient->uExitQualification, rc));
9575	}
9576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9577	return rc;
9578	}
9579
9580
9581	/**
9582	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9583	*/
9584	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9585	{
9586	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9587	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9588	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9589	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9590	AssertRCReturn(rc, rc);
9591
9592	PVM pVM = pVCpu->CTX_SUFF(pVM);
9593	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9594	if (RT_LIKELY(rc == VINF_SUCCESS))
9595	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9596	else
9597	{
9598	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9599	rc = VERR_EM_INTERPRETER;
9600	}
9601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9602	return rc;
9603	}
9604
9605
9606	/**
9607	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9608	*/
9609	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9610	{
9611	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9612	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9613	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9614	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9615	AssertRCReturn(rc, rc);
9616
9617	PVM pVM = pVCpu->CTX_SUFF(pVM);
9618	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9619	rc = VBOXSTRICTRC_VAL(rc2);
9620	if (RT_LIKELY( rc == VINF_SUCCESS
9621	\|\| rc == VINF_EM_HALT))
9622	{
9623	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9624	AssertRCReturn(rc3, rc3);
9625
9626	if ( rc == VINF_EM_HALT
9627	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9628	{
9629	rc = VINF_SUCCESS;
9630	}
9631	}
9632	else
9633	{
9634	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9635	rc = VERR_EM_INTERPRETER;
9636	}
9637	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9638	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9640	return rc;
9641	}
9642
9643
9644	/**
9645	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9646	*/
9647	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9648	{
9649	/*
9650	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9651	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9652	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9653	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9654	*/
9655	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9656	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9657	HMVMX_RETURN_UNEXPECTED_EXIT();
9658	}
9659
9660
9661	/**
9662	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9663	*/
9664	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9665	{
9666	/*
9667	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9668	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9669	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9670	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9671	*/
9672	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9673	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9674	HMVMX_RETURN_UNEXPECTED_EXIT();
9675	}
9676
9677
9678	/**
9679	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9680	*/
9681	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9682	{
9683	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9684	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9685	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9686	HMVMX_RETURN_UNEXPECTED_EXIT();
9687	}
9688
9689
9690	/**
9691	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9692	*/
9693	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9694	{
9695	/*
9696	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9697	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9698	* See Intel spec. 25.3 "Other Causes of VM-exits".
9699	*/
9700	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9701	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9702	HMVMX_RETURN_UNEXPECTED_EXIT();
9703	}
9704
9705
9706	/**
9707	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9708	* VM-exit.
9709	*/
9710	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9711	{
9712	/*
9713	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9714	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9715	*
9716	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9717	* See Intel spec. "23.8 Restrictions on VMX operation".
9718	*/
9719	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9720	return VINF_SUCCESS;
9721	}
9722
9723
9724	/**
9725	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9726	* VM-exit.
9727	*/
9728	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9729	{
9730	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9731	return VINF_EM_RESET;
9732	}
9733
9734
9735	/**
9736	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9737	*/
9738	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9739	{
9740	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9741	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9742	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9743	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9744	AssertRCReturn(rc, rc);
9745
9746	pMixedCtx->rip++;
9747	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9748	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9749	rc = VINF_SUCCESS;
9750	else
9751	rc = VINF_EM_HALT;
9752
9753	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9754	return rc;
9755	}
9756
9757
9758	/**
9759	* VM-exit handler for instructions that result in a #UD exception delivered to
9760	* the guest.
9761	*/
9762	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9763	{
9764	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9765	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9766	return VINF_SUCCESS;
9767	}
9768
9769
9770	/**
9771	* VM-exit handler for expiry of the VMX preemption timer.
9772	*/
9773	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9774	{
9775	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9776
9777	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9778	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9779
9780	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9781	PVM pVM = pVCpu->CTX_SUFF(pVM);
9782	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9784	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9785	}
9786
9787
9788	/**
9789	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9790	*/
9791	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9792	{
9793	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9794
9795	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9796	/** @todo check if XSETBV is supported by the recompiler. */
9797	return VERR_EM_INTERPRETER;
9798	}
9799
9800
9801	/**
9802	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9803	*/
9804	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9805	{
9806	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9807
9808	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9809	/** @todo implement EMInterpretInvpcid() */
9810	return VERR_EM_INTERPRETER;
9811	}
9812
9813
9814	/**
9815	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9816	* Error VM-exit.
9817	*/
9818	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819	{
9820	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9821	AssertRCReturn(rc, rc);
9822
9823	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9824	NOREF(uInvalidReason);
9825
9826	#ifdef VBOX_STRICT
9827	uint32_t uIntrState;
9828	HMVMXHCUINTREG uHCReg;
9829	uint64_t u64Val;
9830	uint32_t u32Val;
9831
9832	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9833	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9834	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9835	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9836	AssertRCReturn(rc, rc);
9837
9838	Log4(("uInvalidReason %u\n", uInvalidReason));
9839	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9840	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9841	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9842	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9843
9844	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9845	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9846	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9847	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9848	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9849	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9850	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9851	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9852	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9853	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9854	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9855	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9856	#else
9857	NOREF(pVmxTransient);
9858	#endif
9859
9860	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9861	return VERR_VMX_INVALID_GUEST_STATE;
9862	}
9863
9864
9865	/**
9866	* VM-exit handler for VM-entry failure due to an MSR-load
9867	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9868	*/
9869	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9870	{
9871	NOREF(pVmxTransient);
9872	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9873	HMVMX_RETURN_UNEXPECTED_EXIT();
9874	}
9875
9876
9877	/**
9878	* VM-exit handler for VM-entry failure due to a machine-check event
9879	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9880	*/
9881	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9882	{
9883	NOREF(pVmxTransient);
9884	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9885	HMVMX_RETURN_UNEXPECTED_EXIT();
9886	}
9887
9888
9889	/**
9890	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9891	* theory.
9892	*/
9893	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9894	{
9895	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9896	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9897	return VERR_VMX_UNDEFINED_EXIT_CODE;
9898	}
9899
9900
9901	/**
9902	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9903	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9904	* Conditional VM-exit.
9905	*/
9906	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9907	{
9908	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9909
9910	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9912	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9913	return VERR_EM_INTERPRETER;
9914	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9915	HMVMX_RETURN_UNEXPECTED_EXIT();
9916	}
9917
9918
9919	/**
9920	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9921	*/
9922	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9923	{
9924	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9925
9926	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9928	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9929	return VERR_EM_INTERPRETER;
9930	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9931	HMVMX_RETURN_UNEXPECTED_EXIT();
9932	}
9933
9934
9935	/**
9936	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9937	*/
9938	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9939	{
9940	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9941
9942	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9943	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9944	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9945	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9946	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9947	{
9948	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9949	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9950	}
9951	AssertRCReturn(rc, rc);
9952	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9953
9954	#ifdef VBOX_STRICT
9955	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9956	{
9957	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9958	{
9959	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9960	HMVMX_RETURN_UNEXPECTED_EXIT();
9961	}
9962	# if HC_ARCH_BITS == 64
9963	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9964	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9965	{
9966	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9967	HMVMX_RETURN_UNEXPECTED_EXIT();
9968	}
9969	# endif
9970	}
9971	#endif
9972
9973	PVM pVM = pVCpu->CTX_SUFF(pVM);
9974	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9975	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9976	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9978
9979	if (RT_LIKELY(rc == VINF_SUCCESS))
9980	{
9981	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9982	Assert(pVmxTransient->cbInstr == 2);
9983	}
9984	return rc;
9985	}
9986
9987
9988	/**
9989	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9990	*/
9991	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9992	{
9993	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9994	PVM pVM = pVCpu->CTX_SUFF(pVM);
9995	int rc = VINF_SUCCESS;
9996
9997	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9998	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9999	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10000	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10001	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10002	{
10003	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10004	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10005	}
10006	AssertRCReturn(rc, rc);
10007	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10008
10009	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10010	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10012
10013	if (RT_LIKELY(rc == VINF_SUCCESS))
10014	{
10015	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10016
10017	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10018	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10019	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10020	{
10021	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10022	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10023	EMInterpretWrmsr() changes it. */
10024	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10025	}
10026	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10027	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10028
10029	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10030	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10031	{
10032	switch (pMixedCtx->ecx)
10033	{
10034	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10035	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10036	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10037	case MSR_K8_FS_BASE: /* no break */
10038	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10039	default:
10040	{
10041	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10042	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10043	#if HC_ARCH_BITS == 64
10044	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10045	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10046	#endif
10047	break;
10048	}
10049	}
10050	}
10051	#ifdef VBOX_STRICT
10052	else
10053	{
10054	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10055	switch (pMixedCtx->ecx)
10056	{
10057	case MSR_IA32_SYSENTER_CS:
10058	case MSR_IA32_SYSENTER_EIP:
10059	case MSR_IA32_SYSENTER_ESP:
10060	case MSR_K8_FS_BASE:
10061	case MSR_K8_GS_BASE:
10062	{
10063	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10064	HMVMX_RETURN_UNEXPECTED_EXIT();
10065	}
10066
10067	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10068	default:
10069	{
10070	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10071	{
10072	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10073	pMixedCtx->ecx));
10074	HMVMX_RETURN_UNEXPECTED_EXIT();
10075	}
10076
10077	#if HC_ARCH_BITS == 64
10078	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10079	{
10080	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10081	HMVMX_RETURN_UNEXPECTED_EXIT();
10082	}
10083	#endif
10084	break;
10085	}
10086	}
10087	}
10088	#endif /* VBOX_STRICT */
10089	}
10090	return rc;
10091	}
10092
10093
10094	/**
10095	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10096	*/
10097	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10098	{
10099	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10100
10101	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10102	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10103	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10104	return VERR_EM_INTERPRETER;
10105	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10106	HMVMX_RETURN_UNEXPECTED_EXIT();
10107	}
10108
10109
10110	/**
10111	* VM-exit handler for when the TPR value is lowered below the specified
10112	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10113	*/
10114	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10115	{
10116	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10117	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10118
10119	/*
10120	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10121	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10122	* resume guest execution.
10123	*/
10124	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10126	return VINF_SUCCESS;
10127	}
10128
10129
10130	/**
10131	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10132	* VM-exit.
10133	*
10134	* @retval VINF_SUCCESS when guest execution can continue.
10135	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10136	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10137	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10138	* recompiler.
10139	*/
10140	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10141	{
10142	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10143	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10144	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10145	AssertRCReturn(rc, rc);
10146
10147	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10148	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10149	PVM pVM = pVCpu->CTX_SUFF(pVM);
10150	switch (uAccessType)
10151	{
10152	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10153	{
10154	#if 0
10155	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10156	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10157	#else
10158	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10159	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10160	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10161	#endif
10162	AssertRCReturn(rc, rc);
10163
10164	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10165	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10166	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10167	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10168
10169	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10170	{
10171	case 0: /* CR0 */
10172	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10173	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10174	break;
10175	case 2: /* CR2 */
10176	/* Nothing to do here, CR2 it's not part of the VMCS. */
10177	break;
10178	case 3: /* CR3 */
10179	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10180	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10181	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10182	break;
10183	case 4: /* CR4 */
10184	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10185	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10186	break;
10187	case 8: /* CR8 */
10188	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10189	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10190	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10191	break;
10192	default:
10193	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10194	break;
10195	}
10196
10197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10198	break;
10199	}
10200
10201	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10202	{
10203	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10204	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10205	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10206	AssertRCReturn(rc, rc);
10207	Assert( !pVM->hm.s.fNestedPaging
10208	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10209	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10210
10211	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10212	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10213	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10214
10215	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10216	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10217	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10218	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10219	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10220	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10221	break;
10222	}
10223
10224	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10225	{
10226	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10227	AssertRCReturn(rc, rc);
10228	rc = EMInterpretCLTS(pVM, pVCpu);
10229	AssertRCReturn(rc, rc);
10230	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10232	Log4(("CRX CLTS write rc=%d\n", rc));
10233	break;
10234	}
10235
10236	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10237	{
10238	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10239	AssertRCReturn(rc, rc);
10240	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10241	if (RT_LIKELY(rc == VINF_SUCCESS))
10242	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10244	Log4(("CRX LMSW write rc=%d\n", rc));
10245	break;
10246	}
10247
10248	default:
10249	{
10250	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10251	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10252	}
10253	}
10254
10255	/* Validate possible error codes. */
10256	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10257	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10258	if (RT_SUCCESS(rc))
10259	{
10260	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10261	AssertRCReturn(rc2, rc2);
10262	}
10263
10264	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10265	return rc;
10266	}
10267
10268
10269	/**
10270	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10271	* VM-exit.
10272	*/
10273	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10274	{
10275	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10276	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10277
10278	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10279	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10280	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10281	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10282	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10283	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10284	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10285	AssertRCReturn(rc2, rc2);
10286
10287	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10288	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10289	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10290	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10291	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10292	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10293	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10294
10295	/* I/O operation lookup arrays. */
10296	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10297	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10298
10299	VBOXSTRICTRC rcStrict;
10300	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10301	const uint32_t cbInstr = pVmxTransient->cbInstr;
10302	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10303	PVM pVM = pVCpu->CTX_SUFF(pVM);
10304	if (fIOString)
10305	{
10306	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10307	/*
10308	* INS/OUTS - I/O String instruction.
10309	*
10310	* Use instruction-information if available, otherwise fall back on
10311	* interpreting the instruction.
10312	*/
10313	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10314	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10315	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10316	{
10317	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10318	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10319	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10320	AssertRCReturn(rc2, rc2);
10321	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10322	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10323	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10324	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10325	if (fIOWrite)
10326	{
10327	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10328	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10329	}
10330	else
10331	{
10332	/*
10333	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10334	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10335	* See Intel Instruction spec. for "INS".
10336	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10337	*/
10338	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10339	}
10340	}
10341	else
10342	{
10343	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10344	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10345	AssertRCReturn(rc2, rc2);
10346	rcStrict = IEMExecOne(pVCpu);
10347	}
10348	/** @todo IEM needs to be setting these flags somehow. */
10349	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10350	fUpdateRipAlready = true;
10351	#else
10352	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10353	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10354	if (RT_SUCCESS(rcStrict))
10355	{
10356	if (fIOWrite)
10357	{
10358	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10359	(DISCPUMODE)pDis->uAddrMode, cbValue);
10360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10361	}
10362	else
10363	{
10364	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10365	(DISCPUMODE)pDis->uAddrMode, cbValue);
10366	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10367	}
10368	}
10369	else
10370	{
10371	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10372	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10373	}
10374	#endif
10375	}
10376	else
10377	{
10378	/*
10379	* IN/OUT - I/O instruction.
10380	*/
10381	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10382	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10383	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10384	if (fIOWrite)
10385	{
10386	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10387	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10388	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10390	}
10391	else
10392	{
10393	uint32_t u32Result = 0;
10394	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10395	if (IOM_SUCCESS(rcStrict))
10396	{
10397	/* Save result of I/O IN instr. in AL/AX/EAX. */
10398	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10399	}
10400	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10401	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10402	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10403	}
10404	}
10405
10406	if (IOM_SUCCESS(rcStrict))
10407	{
10408	if (!fUpdateRipAlready)
10409	{
10410	pMixedCtx->rip += cbInstr;
10411	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10412	}
10413
10414	/*
10415	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10416	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10417	*/
10418	if (fIOString)
10419	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10420
10421	/*
10422	* If any I/O breakpoints are armed, we need to check if one triggered
10423	* and take appropriate action.
10424	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10425	*/
10426	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10427	AssertRCReturn(rc2, rc2);
10428
10429	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10430	* execution engines about whether hyper BPs and such are pending. */
10431	uint32_t const uDr7 = pMixedCtx->dr[7];
10432	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10433	&& X86_DR7_ANY_RW_IO(uDr7)
10434	&& (pMixedCtx->cr4 & X86_CR4_DE))
10435	\|\| DBGFBpIsHwIoArmed(pVM)))
10436	{
10437	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10438
10439	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10440	VMMRZCallRing3Disable(pVCpu);
10441	HM_DISABLE_PREEMPT_IF_NEEDED();
10442
10443	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10444
10445	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10446	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10447	{
10448	/* Raise #DB. */
10449	if (fIsGuestDbgActive)
10450	ASMSetDR6(pMixedCtx->dr[6]);
10451	if (pMixedCtx->dr[7] != uDr7)
10452	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10453
10454	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10455	}
10456	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10457	else if ( rcStrict2 != VINF_SUCCESS
10458	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10459	rcStrict = rcStrict2;
10460
10461	HM_RESTORE_PREEMPT_IF_NEEDED();
10462	VMMRZCallRing3Enable(pVCpu);
10463	}
10464	}
10465
10466	#ifdef DEBUG
10467	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10468	Assert(!fIOWrite);
10469	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10470	Assert(fIOWrite);
10471	else
10472	{
10473	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10474	* statuses, that the VMM device and some others may return. See
10475	* IOM_SUCCESS() for guidance. */
10476	AssertMsg( RT_FAILURE(rcStrict)
10477	\|\| rcStrict == VINF_SUCCESS
10478	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10479	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10480	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10481	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10482	}
10483	#endif
10484
10485	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10486	return VBOXSTRICTRC_TODO(rcStrict);
10487	}
10488
10489
10490	/**
10491	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10492	* VM-exit.
10493	*/
10494	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10495	{
10496	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10497
10498	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10499	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10500	AssertRCReturn(rc, rc);
10501	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10502	{
10503	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10504	AssertRCReturn(rc, rc);
10505	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10506	{
10507	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10508
10509	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10510	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10511	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10512	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10513	{
10514	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10515	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10516
10517	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10518	Assert(!pVCpu->hm.s.Event.fPending);
10519	pVCpu->hm.s.Event.fPending = true;
10520	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10521	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10522	AssertRCReturn(rc, rc);
10523	if (fErrorCodeValid)
10524	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10525	else
10526	pVCpu->hm.s.Event.u32ErrCode = 0;
10527	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10528	&& uVector == X86_XCPT_PF)
10529	{
10530	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10531	}
10532
10533	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10534	}
10535	}
10536	}
10537
10538	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10539	* emulation. */
10540	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10541	return VERR_EM_INTERPRETER;
10542	}
10543
10544
10545	/**
10546	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10547	*/
10548	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10549	{
10550	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10551	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10552	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10553	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10554	AssertRCReturn(rc, rc);
10555	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10556	return VINF_EM_DBG_STEPPED;
10557	}
10558
10559
10560	/**
10561	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10562	*/
10563	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10564	{
10565	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10566
10567	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10568	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10569	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10570	return VINF_SUCCESS;
10571	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10572	return rc;
10573
10574	#if 0
10575	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10576	* just sync the whole thing. */
10577	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10578	#else
10579	/* Aggressive state sync. for now. */
10580	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10581	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10582	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10583	#endif
10584	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10585	AssertRCReturn(rc, rc);
10586
10587	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10588	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10589	switch (uAccessType)
10590	{
10591	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10592	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10593	{
10594	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10595	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10596	{
10597	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10598	}
10599
10600	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10601	GCPhys &= PAGE_BASE_GC_MASK;
10602	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10603	PVM pVM = pVCpu->CTX_SUFF(pVM);
10604	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10605	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10606
10607	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10608	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10609	CPUMCTX2CORE(pMixedCtx), GCPhys);
10610	rc = VBOXSTRICTRC_VAL(rc2);
10611	Log4(("ApicAccess rc=%d\n", rc));
10612	if ( rc == VINF_SUCCESS
10613	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10614	\|\| rc == VERR_PAGE_NOT_PRESENT)
10615	{
10616	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10617	\| HM_CHANGED_GUEST_RSP
10618	\| HM_CHANGED_GUEST_RFLAGS
10619	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10620	rc = VINF_SUCCESS;
10621	}
10622	break;
10623	}
10624
10625	default:
10626	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10627	rc = VINF_EM_RAW_EMULATE_INSTR;
10628	break;
10629	}
10630
10631	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10632	return rc;
10633	}
10634
10635
10636	/**
10637	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10638	* VM-exit.
10639	*/
10640	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10641	{
10642	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10643
10644	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10645	if (pVmxTransient->fWasGuestDebugStateActive)
10646	{
10647	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10648	HMVMX_RETURN_UNEXPECTED_EXIT();
10649	}
10650
10651	int rc = VERR_INTERNAL_ERROR_5;
10652	if ( !DBGFIsStepping(pVCpu)
10653	&& !pVCpu->hm.s.fSingleInstruction
10654	&& !pVmxTransient->fWasHyperDebugStateActive)
10655	{
10656	/* Don't intercept MOV DRx and #DB any more. */
10657	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10658	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10659	AssertRCReturn(rc, rc);
10660
10661	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10662	{
10663	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10664	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10665	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10666	AssertRCReturn(rc, rc);
10667	#endif
10668	}
10669
10670	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10671	VMMRZCallRing3Disable(pVCpu);
10672	HM_DISABLE_PREEMPT_IF_NEEDED();
10673
10674	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10675	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10676	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10677
10678	HM_RESTORE_PREEMPT_IF_NEEDED();
10679	VMMRZCallRing3Enable(pVCpu);
10680
10681	#ifdef VBOX_WITH_STATISTICS
10682	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10683	AssertRCReturn(rc, rc);
10684	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10685	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10686	else
10687	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10688	#endif
10689	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10690	return VINF_SUCCESS;
10691	}
10692
10693	/*
10694	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10695	* Update the segment registers and DR7 from the CPU.
10696	*/
10697	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10698	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10699	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10700	AssertRCReturn(rc, rc);
10701	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10702
10703	PVM pVM = pVCpu->CTX_SUFF(pVM);
10704	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10705	{
10706	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10707	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10708	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10709	if (RT_SUCCESS(rc))
10710	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10712	}
10713	else
10714	{
10715	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10716	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10717	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10719	}
10720
10721	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10722	if (RT_SUCCESS(rc))
10723	{
10724	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10725	AssertRCReturn(rc2, rc2);
10726	}
10727	return rc;
10728	}
10729
10730
10731	/**
10732	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10733	* Conditional VM-exit.
10734	*/
10735	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10736	{
10737	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10738	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10739
10740	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10741	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10742	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10743	return VINF_SUCCESS;
10744	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10745	return rc;
10746
10747	RTGCPHYS GCPhys = 0;
10748	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10749
10750	#if 0
10751	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10752	#else
10753	/* Aggressive state sync. for now. */
10754	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10755	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10756	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10757	#endif
10758	AssertRCReturn(rc, rc);
10759
10760	/*
10761	* If we succeed, resume guest execution.
10762	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10763	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10764	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10765	* weird case. See @bugref{6043}.
10766	*/
10767	PVM pVM = pVCpu->CTX_SUFF(pVM);
10768	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10769	rc = VBOXSTRICTRC_VAL(rc2);
10770	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10771	if ( rc == VINF_SUCCESS
10772	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10773	\|\| rc == VERR_PAGE_NOT_PRESENT)
10774	{
10775	/* Successfully handled MMIO operation. */
10776	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10777	\| HM_CHANGED_GUEST_RSP
10778	\| HM_CHANGED_GUEST_RFLAGS
10779	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10780	rc = VINF_SUCCESS;
10781	}
10782	return rc;
10783	}
10784
10785
10786	/**
10787	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10788	* VM-exit.
10789	*/
10790	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10791	{
10792	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10793	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10794
10795	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10796	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10797	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10798	return VINF_SUCCESS;
10799	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10800	return rc;
10801
10802	RTGCPHYS GCPhys = 0;
10803	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10804	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10805	#if 0
10806	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10807	#else
10808	/* Aggressive state sync. for now. */
10809	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10810	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10811	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10812	#endif
10813	AssertRCReturn(rc, rc);
10814
10815	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10816	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10817
10818	RTGCUINT uErrorCode = 0;
10819	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10820	uErrorCode \|= X86_TRAP_PF_ID;
10821	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10822	uErrorCode \|= X86_TRAP_PF_RW;
10823	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10824	uErrorCode \|= X86_TRAP_PF_P;
10825
10826	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10827
10828	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10829	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10830
10831	/* Handle the pagefault trap for the nested shadow table. */
10832	PVM pVM = pVCpu->CTX_SUFF(pVM);
10833	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10834	TRPMResetTrap(pVCpu);
10835
10836	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10837	if ( rc == VINF_SUCCESS
10838	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10839	\|\| rc == VERR_PAGE_NOT_PRESENT)
10840	{
10841	/* Successfully synced our nested page tables. */
10842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10843	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10844	\| HM_CHANGED_GUEST_RSP
10845	\| HM_CHANGED_GUEST_RFLAGS);
10846	return VINF_SUCCESS;
10847	}
10848
10849	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10850	return rc;
10851	}
10852
10853	/** @} */
10854
10855	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10856	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10857	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10858
10859	/** @name VM-exit exception handlers.
10860	* @{
10861	*/
10862
10863	/**
10864	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10865	*/
10866	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10867	{
10868	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10870
10871	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10872	AssertRCReturn(rc, rc);
10873
10874	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10875	{
10876	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10877	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10878	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10879	AssertRCReturn(rc2, rc2);
10880	return rc;
10881	}
10882
10883	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10884	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10885	return rc;
10886	}
10887
10888
10889	/**
10890	* VM-exit exception handler for #BP (Breakpoint exception).
10891	*/
10892	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10893	{
10894	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10896
10897	/** @todo Try optimize this by not saving the entire guest state unless
10898	* really needed. */
10899	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10900	AssertRCReturn(rc, rc);
10901
10902	PVM pVM = pVCpu->CTX_SUFF(pVM);
10903	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10904	if (rc == VINF_EM_RAW_GUEST_TRAP)
10905	{
10906	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10907	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10908	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10909	AssertRCReturn(rc, rc);
10910
10911	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10912	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10913	}
10914
10915	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10916	return rc;
10917	}
10918
10919
10920	/**
10921	* VM-exit exception handler for #DB (Debug exception).
10922	*/
10923	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10924	{
10925	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10927	Log6(("XcptDB\n"));
10928
10929	/*
10930	* Get the DR6-like values from the exit qualification and pass it to DBGF
10931	* for processing.
10932	*/
10933	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10934	AssertRCReturn(rc, rc);
10935
10936	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10937	uint64_t uDR6 = X86_DR6_INIT_VAL;
10938	uDR6 \|= ( pVmxTransient->uExitQualification
10939	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10940
10941	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10942	if (rc == VINF_EM_RAW_GUEST_TRAP)
10943	{
10944	/*
10945	* The exception was for the guest. Update DR6, DR7.GD and
10946	* IA32_DEBUGCTL.LBR before forwarding it.
10947	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10948	*/
10949	VMMRZCallRing3Disable(pVCpu);
10950	HM_DISABLE_PREEMPT_IF_NEEDED();
10951
10952	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10953	pMixedCtx->dr[6] \|= uDR6;
10954	if (CPUMIsGuestDebugStateActive(pVCpu))
10955	ASMSetDR6(pMixedCtx->dr[6]);
10956
10957	HM_RESTORE_PREEMPT_IF_NEEDED();
10958	VMMRZCallRing3Enable(pVCpu);
10959
10960	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10961	AssertRCReturn(rc, rc);
10962
10963	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10964	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10965
10966	/* Paranoia. */
10967	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10968	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10969
10970	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10971	AssertRCReturn(rc, rc);
10972
10973	/*
10974	* Raise #DB in the guest.
10975	*
10976	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
10977	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
10978	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
10979	*
10980	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
10981	*/
10982	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10983	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10984	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10985	AssertRCReturn(rc, rc);
10986	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10987	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10988	return VINF_SUCCESS;
10989	}
10990
10991	/*
10992	* Not a guest trap, must be a hypervisor related debug event then.
10993	* Update DR6 in case someone is interested in it.
10994	*/
10995	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10996	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10997	CPUMSetHyperDR6(pVCpu, uDR6);
10998
10999	return rc;
11000	}
11001
11002
11003	/**
11004	* VM-exit exception handler for #NM (Device-not-available exception: floating
11005	* point exception).
11006	*/
11007	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11008	{
11009	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11010
11011	/* We require CR0 and EFER. EFER is always up-to-date. */
11012	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11013	AssertRCReturn(rc, rc);
11014
11015	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11016	VMMRZCallRing3Disable(pVCpu);
11017	HM_DISABLE_PREEMPT_IF_NEEDED();
11018
11019	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11020	if (pVmxTransient->fWasGuestFPUStateActive)
11021	{
11022	rc = VINF_EM_RAW_GUEST_TRAP;
11023	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11024	}
11025	else
11026	{
11027	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11028	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11029	#endif
11030	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11031	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11032	}
11033
11034	HM_RESTORE_PREEMPT_IF_NEEDED();
11035	VMMRZCallRing3Enable(pVCpu);
11036
11037	if (rc == VINF_SUCCESS)
11038	{
11039	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11040	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11042	pVCpu->hm.s.fUseGuestFpu = true;
11043	}
11044	else
11045	{
11046	/* Forward #NM to the guest. */
11047	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11048	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11049	AssertRCReturn(rc, rc);
11050	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11051	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11053	}
11054
11055	return VINF_SUCCESS;
11056	}
11057
11058
11059	/**
11060	* VM-exit exception handler for #GP (General-protection exception).
11061	*
11062	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11063	*/
11064	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11065	{
11066	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11067	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11068
11069	int rc = VERR_INTERNAL_ERROR_5;
11070	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11071	{
11072	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11073	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11074	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11075	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11076	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11077	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11078	AssertRCReturn(rc, rc);
11079	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11080	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11081	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11082	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11083	return rc;
11084	#else
11085	/* We don't intercept #GP. */
11086	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11087	NOREF(pVmxTransient);
11088	return VERR_VMX_UNEXPECTED_EXCEPTION;
11089	#endif
11090	}
11091
11092	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11093	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11094
11095	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11096	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11097	AssertRCReturn(rc, rc);
11098
11099	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11100	uint32_t cbOp = 0;
11101	PVM pVM = pVCpu->CTX_SUFF(pVM);
11102	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11103	if (RT_SUCCESS(rc))
11104	{
11105	rc = VINF_SUCCESS;
11106	Assert(cbOp == pDis->cbInstr);
11107	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11108	switch (pDis->pCurInstr->uOpcode)
11109	{
11110	case OP_CLI:
11111	{
11112	pMixedCtx->eflags.Bits.u1IF = 0;
11113	pMixedCtx->rip += pDis->cbInstr;
11114	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11116	break;
11117	}
11118
11119	case OP_STI:
11120	{
11121	pMixedCtx->eflags.Bits.u1IF = 1;
11122	pMixedCtx->rip += pDis->cbInstr;
11123	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11124	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11125	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11126	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11127	break;
11128	}
11129
11130	case OP_HLT:
11131	{
11132	rc = VINF_EM_HALT;
11133	pMixedCtx->rip += pDis->cbInstr;
11134	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11135	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11136	break;
11137	}
11138
11139	case OP_POPF:
11140	{
11141	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11142	uint32_t cbParm = 0;
11143	uint32_t uMask = 0;
11144	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11145	{
11146	cbParm = 4;
11147	uMask = 0xffffffff;
11148	}
11149	else
11150	{
11151	cbParm = 2;
11152	uMask = 0xffff;
11153	}
11154
11155	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11156	RTGCPTR GCPtrStack = 0;
11157	X86EFLAGS Eflags;
11158	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11159	&GCPtrStack);
11160	if (RT_SUCCESS(rc))
11161	{
11162	Assert(sizeof(Eflags.u32) >= cbParm);
11163	Eflags.u32 = 0;
11164	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11165	}
11166	if (RT_FAILURE(rc))
11167	{
11168	rc = VERR_EM_INTERPRETER;
11169	break;
11170	}
11171	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11172	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11173	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11174	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11175	pMixedCtx->esp += cbParm;
11176	pMixedCtx->esp &= uMask;
11177	pMixedCtx->rip += pDis->cbInstr;
11178
11179	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11180	\| HM_CHANGED_GUEST_RSP
11181	\| HM_CHANGED_GUEST_RFLAGS);
11182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11183	break;
11184	}
11185
11186	case OP_PUSHF:
11187	{
11188	uint32_t cbParm = 0;
11189	uint32_t uMask = 0;
11190	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11191	{
11192	cbParm = 4;
11193	uMask = 0xffffffff;
11194	}
11195	else
11196	{
11197	cbParm = 2;
11198	uMask = 0xffff;
11199	}
11200
11201	/* Get the stack pointer & push the contents of eflags onto the stack. */
11202	RTGCPTR GCPtrStack = 0;
11203	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11204	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11205	if (RT_FAILURE(rc))
11206	{
11207	rc = VERR_EM_INTERPRETER;
11208	break;
11209	}
11210	X86EFLAGS Eflags = pMixedCtx->eflags;
11211	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11212	Eflags.Bits.u1RF = 0;
11213	Eflags.Bits.u1VM = 0;
11214
11215	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11216	if (RT_FAILURE(rc))
11217	{
11218	rc = VERR_EM_INTERPRETER;
11219	break;
11220	}
11221	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11222	pMixedCtx->esp -= cbParm;
11223	pMixedCtx->esp &= uMask;
11224	pMixedCtx->rip += pDis->cbInstr;
11225	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11226	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11227	break;
11228	}
11229
11230	case OP_IRET:
11231	{
11232	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11233	* instruction reference. */
11234	RTGCPTR GCPtrStack = 0;
11235	uint32_t uMask = 0xffff;
11236	uint16_t aIretFrame[3];
11237	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11238	{
11239	rc = VERR_EM_INTERPRETER;
11240	break;
11241	}
11242	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11243	&GCPtrStack);
11244	if (RT_SUCCESS(rc))
11245	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11246	if (RT_FAILURE(rc))
11247	{
11248	rc = VERR_EM_INTERPRETER;
11249	break;
11250	}
11251	pMixedCtx->eip = 0;
11252	pMixedCtx->ip = aIretFrame[0];
11253	pMixedCtx->cs.Sel = aIretFrame[1];
11254	pMixedCtx->cs.ValidSel = aIretFrame[1];
11255	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11256	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11257	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11258	pMixedCtx->sp += sizeof(aIretFrame);
11259	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11260	\| HM_CHANGED_GUEST_SEGMENT_REGS
11261	\| HM_CHANGED_GUEST_RSP
11262	\| HM_CHANGED_GUEST_RFLAGS);
11263	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11264	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11265	break;
11266	}
11267
11268	case OP_INT:
11269	{
11270	uint16_t uVector = pDis->Param1.uValue & 0xff;
11271	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11273	break;
11274	}
11275
11276	case OP_INTO:
11277	{
11278	if (pMixedCtx->eflags.Bits.u1OF)
11279	{
11280	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11281	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11282	}
11283	break;
11284	}
11285
11286	default:
11287	{
11288	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11289	EMCODETYPE_SUPERVISOR);
11290	rc = VBOXSTRICTRC_VAL(rc2);
11291	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11292	Log4(("#GP rc=%Rrc\n", rc));
11293	break;
11294	}
11295	}
11296	}
11297	else
11298	rc = VERR_EM_INTERPRETER;
11299
11300	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11301	("#GP Unexpected rc=%Rrc\n", rc));
11302	return rc;
11303	}
11304
11305
11306	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11307	/**
11308	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11309	* the exception reported in the VMX transient structure back into the VM.
11310	*
11311	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11312	* up-to-date.
11313	*/
11314	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11315	{
11316	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11317
11318	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11319	hmR0VmxCheckExitDueToEventDelivery(). */
11320	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11321	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11322	AssertRCReturn(rc, rc);
11323	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11324
11325	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11326	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11327	return VINF_SUCCESS;
11328	}
11329	#endif
11330
11331
11332	/**
11333	* VM-exit exception handler for #PF (Page-fault exception).
11334	*/
11335	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11336	{
11337	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11338	PVM pVM = pVCpu->CTX_SUFF(pVM);
11339	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11340	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11341	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11342	AssertRCReturn(rc, rc);
11343
11344	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11345	if (pVM->hm.s.fNestedPaging)
11346	{
11347	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11348	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11349	{
11350	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11351	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11352	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11353	}
11354	else
11355	{
11356	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11357	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11358	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11359	}
11360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11361	return rc;
11362	}
11363	#else
11364	Assert(!pVM->hm.s.fNestedPaging);
11365	NOREF(pVM);
11366	#endif
11367
11368	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11369	AssertRCReturn(rc, rc);
11370
11371	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11372	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11373
11374	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11375	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11376	(RTGCPTR)pVmxTransient->uExitQualification);
11377
11378	Log4(("#PF: rc=%Rrc\n", rc));
11379	if (rc == VINF_SUCCESS)
11380	{
11381	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11382	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11383	* memory? We don't update the whole state here... */
11384	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11385	\| HM_CHANGED_GUEST_RSP
11386	\| HM_CHANGED_GUEST_RFLAGS
11387	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11388	TRPMResetTrap(pVCpu);
11389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11390	return rc;
11391	}
11392	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11393	{
11394	if (!pVmxTransient->fVectoringPF)
11395	{
11396	/* It's a guest page fault and needs to be reflected to the guest. */
11397	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11398	TRPMResetTrap(pVCpu);
11399	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11400	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11401	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11402	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11403	}
11404	else
11405	{
11406	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11407	TRPMResetTrap(pVCpu);
11408	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11409	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11410	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11411	}
11412
11413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11414	return VINF_SUCCESS;
11415	}
11416
11417	TRPMResetTrap(pVCpu);
11418	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11419	return rc;
11420	}
11421
11422	/** @} */
11423

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49889

Download in other formats: