HMVMXR0.cpp@ 49864

Last change on this file since 49864 was 49864, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Minor nit while checking PAE PDPTEs in hmR0VmxCheckGuestState().
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 470.9 KB

Line
1	/* $Id: HMVMXR0.cpp 49864 2013-12-10 10:26:58Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/x86.h>
23	#include <iprt/asm-amd64-x86.h>
24	#include <iprt/thread.h>
25	#include <iprt/string.h>
26
27	#include "HMInternal.h"
28	#include <VBox/vmm/vm.h>
29	#include "HMVMXR0.h"
30	#include <VBox/vmm/pdmapi.h>
31	#include <VBox/vmm/dbgf.h>
32	#include <VBox/vmm/iem.h>
33	#include <VBox/vmm/iom.h>
34	#include <VBox/vmm/selm.h>
35	#include <VBox/vmm/tm.h>
36	#ifdef VBOX_WITH_REM
37	# include <VBox/vmm/rem.h>
38	#endif
39	#ifdef DEBUG_ramshankar
40	# define HMVMX_SAVE_FULL_GUEST_STATE
41	# define HMVMX_SYNC_FULL_GUEST_STATE
42	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44	# define HMVMX_ALWAYS_TRAP_PF
45	# define HMVMX_ALWAYS_SWAP_FPU_STATE
46	# define HMVMX_ALWAYS_FLUSH_TLB
47	#endif
48
49
50	/*******************************************************************************
51	* Defined Constants And Macros *
52	*******************************************************************************/
53	#if defined(RT_ARCH_AMD64)
54	# define HMVMX_IS_64BIT_HOST_MODE() (true)
55	typedef RTHCUINTREG HMVMXHCUINTREG;
56	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57	extern "C" uint32_t g_fVMXIs64bitHost;
58	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	typedef uint64_t HMVMXHCUINTREG;
60	#else
61	# define HMVMX_IS_64BIT_HOST_MODE() (false)
62	typedef RTHCUINTREG HMVMXHCUINTREG;
63	#endif
64
65	/** Use the function table. */
66	#define HMVMX_USE_FUNCTION_TABLE
67
68	/** Determine which tagged-TLB flush handler to use. */
69	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74	/** @name Updated-guest-state flags.
75	* @{ */
76	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96	\| HMVMX_UPDATED_GUEST_RSP \
97	\| HMVMX_UPDATED_GUEST_RFLAGS \
98	\| HMVMX_UPDATED_GUEST_CR0 \
99	\| HMVMX_UPDATED_GUEST_CR3 \
100	\| HMVMX_UPDATED_GUEST_CR4 \
101	\| HMVMX_UPDATED_GUEST_GDTR \
102	\| HMVMX_UPDATED_GUEST_IDTR \
103	\| HMVMX_UPDATED_GUEST_LDTR \
104	\| HMVMX_UPDATED_GUEST_TR \
105	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106	\| HMVMX_UPDATED_GUEST_DEBUG \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/** @name
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127	/** @} */
128
129	/** @name
130	* States of the VMCS.
131	*
132	* This does not reflect all possible VMCS states but currently only those
133	* needed for maintaining the VMCS consistently even when thread-context hooks
134	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
135	*/
136	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139	/** @} */
140
141	/**
142	* Exception bitmap mask for real-mode guests (real-on-v86).
143	*
144	* We need to intercept all exceptions manually (except #PF). #NM is also
145	* handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146	* even in real-mode if we have Nested Paging support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*******************************************************************************
197	* Structures and Typedefs *
198	*******************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG uEflags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u6Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	} ExitInstrInfo;
247	/** Whether the VM-entry failed or not. */
248	bool fVMEntryFailed;
249	/** Alignment. */
250	uint8_t abAlignment1[3];
251
252	/** The VM-entry interruption-information field. */
253	uint32_t uEntryIntInfo;
254	/** The VM-entry exception error code field. */
255	uint32_t uEntryXcptErrorCode;
256	/** The VM-entry instruction length field. */
257	uint32_t cbEntryInstr;
258
259	/** IDT-vectoring information field. */
260	uint32_t uIdtVectoringInfo;
261	/** IDT-vectoring error code. */
262	uint32_t uIdtVectoringErrorCode;
263
264	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
265	uint32_t fVmcsFieldsRead;
266
267	/** Whether the guest FPU was active at the time of VM-exit. */
268	bool fWasGuestFPUStateActive;
269	/** Whether the guest debug state was active at the time of VM-exit. */
270	bool fWasGuestDebugStateActive;
271	/** Whether the hyper debug state was active at the time of VM-exit. */
272	bool fWasHyperDebugStateActive;
273	/** Whether TSC-offsetting should be setup before VM-entry. */
274	bool fUpdateTscOffsettingAndPreemptTimer;
275	/** Whether the VM-exit was caused by a page-fault during delivery of a
276	* contributory exception or a page-fault. */
277	bool fVectoringPF;
278	} VMXTRANSIENT;
279	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284	/** Pointer to VMX transient state. */
285	typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288	/**
289	* MSR-bitmap read permissions.
290	*/
291	typedef enum VMXMSREXITREAD
292	{
293	/** Reading this MSR causes a VM-exit. */
294	VMXMSREXIT_INTERCEPT_READ = 0xb,
295	/** Reading this MSR does not cause a VM-exit. */
296	VMXMSREXIT_PASSTHRU_READ
297	} VMXMSREXITREAD;
298	/** Pointer to MSR-bitmap read permissions. */
299	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301	/**
302	* MSR-bitmap write permissions.
303	*/
304	typedef enum VMXMSREXITWRITE
305	{
306	/** Writing to this MSR causes a VM-exit. */
307	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308	/** Writing to this MSR does not cause a VM-exit. */
309	VMXMSREXIT_PASSTHRU_WRITE
310	} VMXMSREXITWRITE;
311	/** Pointer to MSR-bitmap write permissions. */
312	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315	/**
316	* VMX VM-exit handler.
317	*
318	* @returns VBox status code.
319	* @param pVCpu Pointer to the VMCPU.
320	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
321	* out-of-sync. Make sure to update the required
322	* fields before using them.
323	* @param pVmxTransient Pointer to the VMX-transient structure.
324	*/
325	#ifndef HMVMX_USE_FUNCTION_TABLE
326	typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327	#else
328	typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329	/** Pointer to VM-exit handler. */
330	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331	#endif
332
333
334	/*******************************************************************************
335	* Internal Functions *
336	*******************************************************************************/
337	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343	#endif
344	#ifndef HMVMX_USE_FUNCTION_TABLE
345	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346	# define HMVMX_EXIT_DECL static int
347	#else
348	# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349	#endif
350
351	/** @name VM-exit handlers.
352	* @{
353	*/
354	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357	static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358	static FNVMXEXITHANDLER hmR0VmxExitSipi;
359	static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360	static FNVMXEXITHANDLER hmR0VmxExitSmi;
361	static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362	static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366	static FNVMXEXITHANDLER hmR0VmxExitHlt;
367	static FNVMXEXITHANDLER hmR0VmxExitInvd;
368	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371	static FNVMXEXITHANDLER hmR0VmxExitRsm;
372	static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378	static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379	static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380	static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381	static FNVMXEXITHANDLER hmR0VmxExitMwait;
382	static FNVMXEXITHANDLER hmR0VmxExitMtf;
383	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384	static FNVMXEXITHANDLER hmR0VmxExitPause;
385	static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386	static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394	static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398	/** @} */
399
400	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408	#endif
409	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411	/*******************************************************************************
412	* Global Variables *
413	*******************************************************************************/
414	#ifdef HMVMX_USE_FUNCTION_TABLE
415
416	/**
417	* VMX_EXIT dispatch table.
418	*/
419	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420	{
421	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461	/* 40 UNDEFINED */ hmR0VmxExitPause,
462	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481	};
482	#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484	#ifdef VBOX_STRICT
485	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486	{
487	/* 0 */ "(Not Used)",
488	/* 1 */ "VMCALL executed in VMX root operation.",
489	/* 2 */ "VMCLEAR with invalid physical address.",
490	/* 3 */ "VMCLEAR with VMXON pointer.",
491	/* 4 */ "VMLAUNCH with non-clear VMCS.",
492	/* 5 */ "VMRESUME with non-launched VMCS.",
493	/* 6 */ "VMRESUME after VMXOFF",
494	/* 7 */ "VM entry with invalid control fields.",
495	/* 8 */ "VM entry with invalid host state fields.",
496	/* 9 */ "VMPTRLD with invalid physical address.",
497	/* 10 */ "VMPTRLD with VMXON pointer.",
498	/* 11 */ "VMPTRLD with incorrect revision identifier.",
499	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500	/* 13 */ "VMWRITE to read-only VMCS component.",
501	/* 14 */ "(Not Used)",
502	/* 15 */ "VMXON executed in VMX root operation.",
503	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
504	/* 17 */ "VM entry with non-launched executing VMCS.",
505	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506	/* 19 */ "VMCALL with non-clear VMCS.",
507	/* 20 */ "VMCALL with invalid VM-exit control fields.",
508	/* 21 */ "(Not Used)",
509	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511	/* 24 */ "VMCALL with invalid SMM-monitor features.",
512	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513	/* 26 */ "VM entry with events blocked by MOV SS.",
514	/* 27 */ "(Not Used)",
515	/* 28 */ "Invalid operand to INVEPT/INVVPID."
516	};
517	#endif /* VBOX_STRICT */
518
519
520
521	/**
522	* Updates the VM's last error record. If there was a VMX instruction error,
523	* reads the error data from the VMCS and updates VCPU's last error record as
524	* well.
525	*
526	* @param pVM Pointer to the VM.
527	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528	* VERR_VMX_UNABLE_TO_START_VM or
529	* VERR_VMX_INVALID_VMCS_FIELD).
530	* @param rc The error code.
531	*/
532	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533	{
534	AssertPtr(pVM);
535	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
537	{
538	AssertPtrReturnVoid(pVCpu);
539	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540	}
541	pVM->hm.s.lLastError = rc;
542	}
543
544
545	/**
546	* Reads the VM-entry interruption-information field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*
552	* @remarks No-long-jump zone!!!
553	*/
554	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555	{
556	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557	AssertRCReturn(rc, rc);
558	return VINF_SUCCESS;
559	}
560
561
562	/**
563	* Reads the VM-entry exception error code field from the VMCS into
564	* the VMX transient structure.
565	*
566	* @returns VBox status code.
567	* @param pVmxTransient Pointer to the VMX transient structure.
568	*
569	* @remarks No-long-jump zone!!!
570	*/
571	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572	{
573	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574	AssertRCReturn(rc, rc);
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the VM-entry exception error code field from the VMCS into
581	* the VMX transient structure.
582	*
583	* @returns VBox status code.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*
586	* @remarks No-long-jump zone!!!
587	*/
588	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589	{
590	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591	AssertRCReturn(rc, rc);
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Reads the VM-exit interruption-information field from the VMCS into the VMX
598	* transient structure.
599	*
600	* @returns VBox status code.
601	* @param pVmxTransient Pointer to the VMX transient structure.
602	*/
603	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the VM-exit interruption error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Reads the VM-exit instruction length field from the VMCS into the VMX
636	* transient structure.
637	*
638	* @returns VBox status code.
639	* @param pVCpu Pointer to the VMCPU.
640	* @param pVmxTransient Pointer to the VMX transient structure.
641	*/
642	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643	{
644	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645	{
646	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647	AssertRCReturn(rc, rc);
648	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649	}
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Reads the VM-exit instruction-information field from the VMCS into
656	* the VMX transient structure.
657	*
658	* @returns VBox status code.
659	* @param pVmxTransient Pointer to the VMX transient structure.
660	*/
661	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662	{
663	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664	{
665	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666	AssertRCReturn(rc, rc);
667	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668	}
669	return VINF_SUCCESS;
670	}
671
672
673	/**
674	* Reads the exit qualification from the VMCS into the VMX transient structure.
675	*
676	* @returns VBox status code.
677	* @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678	* case).
679	* @param pVmxTransient Pointer to the VMX transient structure.
680	*/
681	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682	{
683	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684	{
685	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686	AssertRCReturn(rc, rc);
687	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688	}
689	return VINF_SUCCESS;
690	}
691
692
693	/**
694	* Reads the IDT-vectoring information field from the VMCS into the VMX
695	* transient structure.
696	*
697	* @returns VBox status code.
698	* @param pVmxTransient Pointer to the VMX transient structure.
699	*
700	* @remarks No-long-jump zone!!!
701	*/
702	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703	{
704	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705	{
706	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707	AssertRCReturn(rc, rc);
708	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709	}
710	return VINF_SUCCESS;
711	}
712
713
714	/**
715	* Reads the IDT-vectoring error code from the VMCS into the VMX
716	* transient structure.
717	*
718	* @returns VBox status code.
719	* @param pVmxTransient Pointer to the VMX transient structure.
720	*/
721	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722	{
723	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724	{
725	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726	AssertRCReturn(rc, rc);
727	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728	}
729	return VINF_SUCCESS;
730	}
731
732
733	/**
734	* Enters VMX root mode operation on the current CPU.
735	*
736	* @returns VBox status code.
737	* @param pVM Pointer to the VM (optional, can be NULL, after
738	* a resume).
739	* @param HCPhysCpuPage Physical address of the VMXON region.
740	* @param pvCpuPage Pointer to the VMXON region.
741	*/
742	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743	{
744	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
745	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
746	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
747
748	if (pVM)
749	{
750	/* Write the VMCS revision dword to the VMXON region. */
751	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
752	}
753
754	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
755	RTCCUINTREG uEflags = ASMIntDisableFlags();
756
757	/* Enable the VMX bit in CR4 if necessary. */
758	RTCCUINTREG uCr4 = ASMGetCR4();
759	if (!(uCr4 & X86_CR4_VMXE))
760	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
761
762	/* Enter VMX root mode. */
763	int rc = VMXEnable(HCPhysCpuPage);
764	if (RT_FAILURE(rc))
765	ASMSetCR4(uCr4);
766
767	/* Restore interrupts. */
768	ASMSetFlags(uEflags);
769	return rc;
770	}
771
772
773	/**
774	* Exits VMX root mode operation on the current CPU.
775	*
776	* @returns VBox status code.
777	*/
778	static int hmR0VmxLeaveRootMode(void)
779	{
780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
781
782	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
783	RTCCUINTREG uEflags = ASMIntDisableFlags();
784
785	/* If we're for some reason not in VMX root mode, then don't leave it. */
786	RTCCUINTREG uHostCR4 = ASMGetCR4();
787
788	int rc;
789	if (uHostCR4 & X86_CR4_VMXE)
790	{
791	/* Exit VMX root mode and clear the VMX bit in CR4. */
792	VMXDisable();
793	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
794	rc = VINF_SUCCESS;
795	}
796	else
797	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
798
799	/* Restore interrupts. */
800	ASMSetFlags(uEflags);
801	return rc;
802	}
803
804
805	/**
806	* Allocates and maps one physically contiguous page. The allocated page is
807	* zero'd out. (Used by various VT-x structures).
808	*
809	* @returns IPRT status code.
810	* @param pMemObj Pointer to the ring-0 memory object.
811	* @param ppVirt Where to store the virtual address of the
812	* allocation.
813	* @param pPhys Where to store the physical address of the
814	* allocation.
815	*/
816	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
817	{
818	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
819	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
820	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
821
822	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
823	if (RT_FAILURE(rc))
824	return rc;
825	ppVirt = RTR0MemObjAddress(pMemObj);
826	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
827	ASMMemZero32(*ppVirt, PAGE_SIZE);
828	return VINF_SUCCESS;
829	}
830
831
832	/**
833	* Frees and unmaps an allocated physical page.
834	*
835	* @param pMemObj Pointer to the ring-0 memory object.
836	* @param ppVirt Where to re-initialize the virtual address of
837	* allocation as 0.
838	* @param pHCPhys Where to re-initialize the physical address of the
839	* allocation as 0.
840	*/
841	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
842	{
843	AssertPtr(pMemObj);
844	AssertPtr(ppVirt);
845	AssertPtr(pHCPhys);
846	if (*pMemObj != NIL_RTR0MEMOBJ)
847	{
848	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
849	AssertRC(rc);
850	*pMemObj = NIL_RTR0MEMOBJ;
851	*ppVirt = 0;
852	*pHCPhys = 0;
853	}
854	}
855
856
857	/**
858	* Worker function to free VT-x related structures.
859	*
860	* @returns IPRT status code.
861	* @param pVM Pointer to the VM.
862	*/
863	static void hmR0VmxStructsFree(PVM pVM)
864	{
865	for (VMCPUID i = 0; i < pVM->cCpus; i++)
866	{
867	PVMCPU pVCpu = &pVM->aCpus[i];
868	AssertPtr(pVCpu);
869
870	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
871	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
872
873	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
874	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
875
876	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
877	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
878	}
879
880	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
881	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
882	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
883	#endif
884	}
885
886
887	/**
888	* Worker function to allocate VT-x related VM structures.
889	*
890	* @returns IPRT status code.
891	* @param pVM Pointer to the VM.
892	*/
893	static int hmR0VmxStructsAlloc(PVM pVM)
894	{
895	/*
896	* Initialize members up-front so we can cleanup properly on allocation failure.
897	*/
898	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
899	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
900	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
901	pVM->hm.s.vmx.HCPhys##a_Name = 0;
902
903	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
904	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
907
908	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
910	#endif
911	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
912
913	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
914	for (VMCPUID i = 0; i < pVM->cCpus; i++)
915	{
916	PVMCPU pVCpu = &pVM->aCpus[i];
917	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
918	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
919	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
920	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
921	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
922	}
923	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
924	#undef VMXLOCAL_INIT_VM_MEMOBJ
925
926	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
927	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
928	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
929	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
930
931	/*
932	* Allocate all the VT-x structures.
933	*/
934	int rc = VINF_SUCCESS;
935	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
936	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
937	if (RT_FAILURE(rc))
938	goto cleanup;
939	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
940	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
941	#endif
942
943	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
944	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
945	{
946	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
947	&pVM->hm.s.vmx.HCPhysApicAccess);
948	if (RT_FAILURE(rc))
949	goto cleanup;
950	}
951
952	/*
953	* Initialize per-VCPU VT-x structures.
954	*/
955	for (VMCPUID i = 0; i < pVM->cCpus; i++)
956	{
957	PVMCPU pVCpu = &pVM->aCpus[i];
958	AssertPtr(pVCpu);
959
960	/* Allocate the VM control structure (VMCS). */
961	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
962	if (RT_FAILURE(rc))
963	goto cleanup;
964
965	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
966	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
967	{
968	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
969	&pVCpu->hm.s.vmx.HCPhysVirtApic);
970	if (RT_FAILURE(rc))
971	goto cleanup;
972	}
973
974	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
975	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
976	{
977	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
978	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
979	if (RT_FAILURE(rc))
980	goto cleanup;
981	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
982	}
983
984	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
985	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
986	if (RT_FAILURE(rc))
987	goto cleanup;
988
989	/* Allocate the VM-exit MSR-load page for the host MSRs. */
990	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
991	if (RT_FAILURE(rc))
992	goto cleanup;
993	}
994
995	return VINF_SUCCESS;
996
997	cleanup:
998	hmR0VmxStructsFree(pVM);
999	return rc;
1000	}
1001
1002
1003	/**
1004	* Does global VT-x initialization (called during module initialization).
1005	*
1006	* @returns VBox status code.
1007	*/
1008	VMMR0DECL(int) VMXR0GlobalInit(void)
1009	{
1010	#ifdef HMVMX_USE_FUNCTION_TABLE
1011	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1012	# ifdef VBOX_STRICT
1013	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1014	Assert(g_apfnVMExitHandlers[i]);
1015	# endif
1016	#endif
1017	return VINF_SUCCESS;
1018	}
1019
1020
1021	/**
1022	* Does global VT-x termination (called during module termination).
1023	*/
1024	VMMR0DECL(void) VMXR0GlobalTerm()
1025	{
1026	/* Nothing to do currently. */
1027	}
1028
1029
1030	/**
1031	* Sets up and activates VT-x on the current CPU.
1032	*
1033	* @returns VBox status code.
1034	* @param pCpu Pointer to the global CPU info struct.
1035	* @param pVM Pointer to the VM (can be NULL after a host resume
1036	* operation).
1037	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1038	* fEnabledByHost is true).
1039	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1040	* @a fEnabledByHost is true).
1041	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1042	* enable VT-x on the host.
1043	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1044	*/
1045	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1046	void *pvMsrs)
1047	{
1048	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1049	AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1050	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1051
1052	/* Enable VT-x if it's not already enabled by the host. */
1053	if (!fEnabledByHost)
1054	{
1055	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1056	if (RT_FAILURE(rc))
1057	return rc;
1058	}
1059
1060	/*
1061	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1062	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1063	*/
1064	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1065	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1066	{
1067	hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1068	pCpu->fFlushAsidBeforeUse = false;
1069	}
1070	else
1071	pCpu->fFlushAsidBeforeUse = true;
1072
1073	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1074	++pCpu->cTlbFlushes;
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Deactivates VT-x on the current CPU.
1082	*
1083	* @returns VBox status code.
1084	* @param pCpu Pointer to the global CPU info struct.
1085	* @param pvCpuPage Pointer to the VMXON region.
1086	* @param HCPhysCpuPage Physical address of the VMXON region.
1087	*
1088	* @remarks This function should never be called when SUPR0EnableVTx() or
1089	* similar was used to enable VT-x on the host.
1090	*/
1091	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1092	{
1093	NOREF(pCpu);
1094	NOREF(pvCpuPage);
1095	NOREF(HCPhysCpuPage);
1096
1097	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1098	return hmR0VmxLeaveRootMode();
1099	}
1100
1101
1102	/**
1103	* Sets the permission bits for the specified MSR in the MSR bitmap.
1104	*
1105	* @param pVCpu Pointer to the VMCPU.
1106	* @param uMSR The MSR value.
1107	* @param enmRead Whether reading this MSR causes a VM-exit.
1108	* @param enmWrite Whether writing this MSR causes a VM-exit.
1109	*/
1110	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1111	{
1112	int32_t iBit;
1113	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1114
1115	/*
1116	* Layout:
1117	* 0x000 - 0x3ff - Low MSR read bits
1118	* 0x400 - 0x7ff - High MSR read bits
1119	* 0x800 - 0xbff - Low MSR write bits
1120	* 0xc00 - 0xfff - High MSR write bits
1121	*/
1122	if (uMsr <= 0x00001FFF)
1123	iBit = uMsr;
1124	else if ( uMsr >= 0xC0000000
1125	&& uMsr <= 0xC0001FFF)
1126	{
1127	iBit = (uMsr - 0xC0000000);
1128	pbMsrBitmap += 0x400;
1129	}
1130	else
1131	{
1132	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1133	return;
1134	}
1135
1136	Assert(iBit <= 0x1fff);
1137	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1138	ASMBitSet(pbMsrBitmap, iBit);
1139	else
1140	ASMBitClear(pbMsrBitmap, iBit);
1141
1142	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1143	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1144	else
1145	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1146	}
1147
1148
1149	#ifdef VBOX_STRICT
1150	/**
1151	* Gets the permission bits for the specified MSR in the MSR bitmap.
1152	*
1153	* @returns VBox status code.
1154	* @retval VINF_SUCCESS if the specified MSR is found.
1155	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1156	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1157	*
1158	* @param pVCpu Pointer to the VMCPU.
1159	* @param uMsr The MSR.
1160	* @param penmRead Where to store the read permissions.
1161	* @param penmWrite Where to store the write permissions.
1162	*/
1163	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1164	{
1165	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1166	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1167	int32_t iBit;
1168	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170	/* See hmR0VmxSetMsrPermission() for the layout. */
1171	if (uMsr <= 0x00001FFF)
1172	iBit = uMsr;
1173	else if ( uMsr >= 0xC0000000
1174	&& uMsr <= 0xC0001FFF)
1175	{
1176	iBit = (uMsr - 0xC0000000);
1177	pbMsrBitmap += 0x400;
1178	}
1179	else
1180	{
1181	AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182	return VERR_NOT_SUPPORTED;
1183	}
1184
1185	Assert(iBit <= 0x1fff);
1186	if (ASMBitTest(pbMsrBitmap, iBit))
1187	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1188	else
1189	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1190
1191	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1192	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1193	else
1194	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1195	return VINF_SUCCESS;
1196	}
1197	#endif /* VBOX_STRICT */
1198
1199
1200	/**
1201	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1202	* area.
1203	*
1204	* @returns VBox status code.
1205	* @param pVCpu Pointer to the VMCPU.
1206	* @param cMsrs The number of MSRs.
1207	*/
1208	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1209	{
1210	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1211	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1212	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1213	{
1214	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1215	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1216	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1217	}
1218
1219	/* Update number of guest MSRs to load/store across the world-switch. */
1220	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1221	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222
1223	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1224	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226	/* Update the VCPU's copy of the MSR count. */
1227	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1228
1229	return VINF_SUCCESS;
1230	}
1231
1232
1233	/**
1234	* Adds a new (or updates the value of an existing) guest/host MSR
1235	* pair to be swapped during the world-switch as part of the
1236	* auto-load/store MSR area in the VMCS.
1237	*
1238	* @returns VBox status code.
1239	* @param pVCpu Pointer to the VMCPU.
1240	* @param uMsr The MSR.
1241	* @param uGuestMsr Value of the guest MSR.
1242	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1243	* necessary.
1244	*/
1245	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1246	{
1247	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1248	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1249	uint32_t i;
1250	for (i = 0; i < cMsrs; i++)
1251	{
1252	if (pGuestMsr->u32Msr == uMsr)
1253	break;
1254	pGuestMsr++;
1255	}
1256
1257	bool fAdded = false;
1258	if (i == cMsrs)
1259	{
1260	++cMsrs;
1261	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1262	AssertRCReturn(rc, rc);
1263
1264	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1265	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1266	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1267
1268	fAdded = true;
1269	}
1270
1271	/* Update the MSR values in the auto-load/store MSR area. */
1272	pGuestMsr->u32Msr = uMsr;
1273	pGuestMsr->u64Value = uGuestMsrValue;
1274
1275	/* Create/update the MSR slot in the host MSR area. */
1276	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1277	pHostMsr += i;
1278	pHostMsr->u32Msr = uMsr;
1279
1280	/*
1281	* Update the host MSR only when requested by the caller AND when we're
1282	* adding it to the auto-load/store area. Otherwise, it would have been
1283	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1284	*/
1285	if ( fAdded
1286	&& fUpdateHostMsr)
1287	{
1288	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1289	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1290	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1291	}
1292
1293	return VINF_SUCCESS;
1294	}
1295
1296
1297	/**
1298	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1299	* auto-load/store MSR area in the VMCS.
1300	*
1301	* Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1302	* area.
1303	*
1304	* @returns VBox status code.
1305	* @param pVCpu Pointer to the VMCPU.
1306	* @param uMsr The MSR.
1307	*/
1308	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1309	{
1310	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1311	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1312	for (uint32_t i = 0; i < cMsrs; i++)
1313	{
1314	/* Find the MSR. */
1315	if (pGuestMsr->u32Msr == uMsr)
1316	{
1317	/* If it's the last MSR, simply reduce the count. */
1318	if (i == cMsrs - 1)
1319	{
1320	--cMsrs;
1321	break;
1322	}
1323
1324	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1325	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326	pLastGuestMsr += cMsrs;
1327	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1328	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1329
1330	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1331	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332	pLastHostMsr += cMsrs;
1333	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1334	pHostMsr->u64Value = pLastHostMsr->u64Value;
1335	--cMsrs;
1336	break;
1337	}
1338	pGuestMsr++;
1339	}
1340
1341	/* Update the VMCS if the count changed (meaning the MSR was found). */
1342	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1343	{
1344	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1345	AssertRCReturn(rc, rc);
1346
1347	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1349	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1350	}
1351
1352	return VINF_SUCCESS;
1353	}
1354
1355
1356	/**
1357	* Checks if the specified guest MSR is part of the auto-load/store area in
1358	* the VMCS.
1359	*
1360	* @returns true if found, false otherwise.
1361	* @param pVCpu Pointer to the VMCPU.
1362	* @param uMsr The MSR to find.
1363	*/
1364	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1365	{
1366	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368
1369	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1370	{
1371	if (pGuestMsr->u32Msr == uMsr)
1372	return true;
1373	}
1374	return false;
1375	}
1376
1377
1378	/**
1379	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1380	*
1381	* @param pVCpu Pointer to the VMCPU.
1382	*
1383	* @remarks No-long-jump zone!!!
1384	*/
1385	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1386	{
1387	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1388	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1389	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1393	{
1394	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1395	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1396	}
1397
1398	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1399	}
1400
1401
1402	#if HC_ARCH_BITS == 64
1403	/**
1404	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1405	* perform lazy restoration of the host MSRs while leaving VT-x.
1406	*
1407	* @param pVCpu Pointer to the VMCPU.
1408	*
1409	* @remarks No-long-jump zone!!!
1410	*/
1411	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1412	{
1413	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1414
1415	/*
1416	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1417	*/
1418	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1419	{
1420	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1421	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1422	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1423	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1424	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1425	}
1426	}
1427
1428
1429	/**
1430	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1431	* lazily while leaving VT-x.
1432	*
1433	* @returns true if it does, false otherwise.
1434	* @param pVCpu Pointer to the VMCPU.
1435	* @param uMsr The MSR to check.
1436	*/
1437	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1438	{
1439	NOREF(pVCpu);
1440	switch (uMsr)
1441	{
1442	case MSR_K8_LSTAR:
1443	case MSR_K6_STAR:
1444	case MSR_K8_SF_MASK:
1445	case MSR_K8_KERNEL_GS_BASE:
1446	return true;
1447	}
1448	return false;
1449	}
1450
1451
1452	/**
1453	* Saves a set of guests MSRs back into the guest-CPU context.
1454	*
1455	* @param pVCpu Pointer to the VMCPU.
1456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1457	* out-of-sync. Make sure to update the required fields
1458	* before using them.
1459	*
1460	* @remarks No-long-jump zone!!!
1461	*/
1462	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1463	{
1464	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1465	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1466
1467	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1468	{
1469	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1471	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1472	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1473	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1474	}
1475	}
1476
1477
1478	/**
1479	* Loads a set of guests MSRs to allow read/passthru to the guest.
1480	*
1481	* The name of this function is slightly confusing. This function does NOT
1482	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1483	* common prefix for functions dealing with "lazy restoration" of the shared
1484	* MSRs.
1485	*
1486	* @param pVCpu Pointer to the VMCPU.
1487	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488	* out-of-sync. Make sure to update the required fields
1489	* before using them.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1499	if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1500	{
1501	#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1502	do { \
1503	if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1504	ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1505	else \
1506	Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1507	} while (0)
1508
1509	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1510	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1511	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1512	VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1513	#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1514	}
1515	else
1516	{
1517	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1518	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1519	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1520	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1521	}
1522	pVCpu->hm.s.vmx.fRestoreHostMsrs \|= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1523	}
1524
1525
1526	/**
1527	* Performs lazy restoration of the set of host MSRs if they were previously
1528	* loaded with guest MSR values.
1529	*
1530	* @param pVCpu Pointer to the VMCPU.
1531	*
1532	* @remarks No-long-jump zone!!!
1533	* @remarks The guest MSRs should have been saved back into the guest-CPU
1534	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1535	*/
1536	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1537	{
1538	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1539	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1540
1541	if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1542	{
1543	Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1544	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1545	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1546	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1547	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1548	}
1549	pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST \| VMX_RESTORE_HOST_MSR_SAVED_HOST);
1550	}
1551	#endif /* HC_ARCH_BITS == 64 */
1552
1553
1554	#ifdef VBOX_STRICT
1555	/**
1556	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1557	* VMCS are correct.
1558	*
1559	* @param pVCpu Pointer to the VMCPU.
1560	*/
1561	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1562	{
1563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564
1565	/* Verify MSR counts in the VMCS are what we think it should be. */
1566	uint32_t cMsrs;
1567	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1568	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1569
1570	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1571	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1572
1573	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1574	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1575
1576	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1577	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1578	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1579	{
1580	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1581	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1582	pGuestMsr->u32Msr));
1583
1584	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1585	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1586	pHostMsr->u64Value, u64Msr));
1587
1588	/* Verify that the permissions are as expected in the MSR bitmap. */
1589	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1590	{
1591	VMXMSREXITREAD enmRead;
1592	VMXMSREXITWRITE enmWrite;
1593	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1594	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1595	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1596	pGuestMsr->u32Msr));
1597	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1598	pGuestMsr->u32Msr));
1599	}
1600	}
1601	}
1602	# endif /* VBOX_STRICT */
1603
1604
1605	/**
1606	* Flushes the TLB using EPT.
1607	*
1608	* @returns VBox status code.
1609	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1610	* enmFlush).
1611	* @param enmFlush Type of flush.
1612	*
1613	* @remarks Caller is responsible for making sure this function is called only
1614	* when NestedPaging is supported and providing @a enmFlush that is
1615	* supported by the CPU.
1616	* @remarks Can be called with interrupts disabled.
1617	*/
1618	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1619	{
1620	uint64_t au64Descriptor[2];
1621	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1622	au64Descriptor[0] = 0;
1623	else
1624	{
1625	Assert(pVCpu);
1626	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1627	}
1628	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1629
1630	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1631	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1632	rc));
1633	if ( RT_SUCCESS(rc)
1634	&& pVCpu)
1635	{
1636	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1637	}
1638	}
1639
1640
1641	/**
1642	* Flushes the TLB using VPID.
1643	*
1644	* @returns VBox status code.
1645	* @param pVM Pointer to the VM.
1646	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1647	* enmFlush).
1648	* @param enmFlush Type of flush.
1649	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1650	* on @a enmFlush).
1651	*
1652	* @remarks Can be called with interrupts disabled.
1653	*/
1654	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1655	{
1656	NOREF(pVM);
1657	AssertPtr(pVM);
1658	Assert(pVM->hm.s.vmx.fVpid);
1659
1660	uint64_t au64Descriptor[2];
1661	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1662	{
1663	au64Descriptor[0] = 0;
1664	au64Descriptor[1] = 0;
1665	}
1666	else
1667	{
1668	AssertPtr(pVCpu);
1669	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1670	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1671	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1672	au64Descriptor[1] = GCPtr;
1673	}
1674
1675	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1676	AssertMsg(rc == VINF_SUCCESS,
1677	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1678	if ( RT_SUCCESS(rc)
1679	&& pVCpu)
1680	{
1681	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1682	}
1683	}
1684
1685
1686	/**
1687	* Invalidates a guest page by guest virtual address. Only relevant for
1688	* EPT/VPID, otherwise there is nothing really to invalidate.
1689	*
1690	* @returns VBox status code.
1691	* @param pVM Pointer to the VM.
1692	* @param pVCpu Pointer to the VMCPU.
1693	* @param GCVirt Guest virtual address of the page to invalidate.
1694	*/
1695	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1696	{
1697	AssertPtr(pVM);
1698	AssertPtr(pVCpu);
1699	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1700
1701	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1702	if (!fFlushPending)
1703	{
1704	/*
1705	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1706	* See @bugref{6043} and @bugref{6177}.
1707	*
1708	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1709	* function maybe called in a loop with individual addresses.
1710	*/
1711	if (pVM->hm.s.vmx.fVpid)
1712	{
1713	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1714	{
1715	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1716	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1717	}
1718	else
1719	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1720	}
1721	else if (pVM->hm.s.fNestedPaging)
1722	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1723	}
1724
1725	return VINF_SUCCESS;
1726	}
1727
1728
1729	/**
1730	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1731	* otherwise there is nothing really to invalidate.
1732	*
1733	* @returns VBox status code.
1734	* @param pVM Pointer to the VM.
1735	* @param pVCpu Pointer to the VMCPU.
1736	* @param GCPhys Guest physical address of the page to invalidate.
1737	*/
1738	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1739	{
1740	NOREF(pVM); NOREF(GCPhys);
1741	LogFlowFunc(("%RGp\n", GCPhys));
1742
1743	/*
1744	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1745	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1746	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1747	*/
1748	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1749	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1750	return VINF_SUCCESS;
1751	}
1752
1753
1754	/**
1755	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1756	* case where neither EPT nor VPID is supported by the CPU.
1757	*
1758	* @param pVM Pointer to the VM.
1759	* @param pVCpu Pointer to the VMCPU.
1760	* @param pCpu Pointer to the global HM struct.
1761	*
1762	* @remarks Called with interrupts disabled.
1763	*/
1764	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1765	{
1766	AssertPtr(pVCpu);
1767	AssertPtr(pCpu);
1768	NOREF(pVM);
1769
1770	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1771
1772	/** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1773	#if 0
1774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1775	pVCpu->hm.s.TlbShootdown.cPages = 0;
1776	#endif
1777
1778	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1779	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1780	pVCpu->hm.s.fForceTLBFlush = false;
1781	return;
1782	}
1783
1784
1785	/**
1786	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1787	*
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	* @param pCpu Pointer to the global HM CPU struct.
1791	* @remarks All references to "ASID" in this function pertains to "VPID" in
1792	* Intel's nomenclature. The reason is, to avoid confusion in compare
1793	* statements since the host-CPU copies are named "ASID".
1794	*
1795	* @remarks Called with interrupts disabled.
1796	*/
1797	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1798	{
1799	#ifdef VBOX_WITH_STATISTICS
1800	bool fTlbFlushed = false;
1801	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1802	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1803	if (!fTlbFlushed) \
1804	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1805	} while (0)
1806	#else
1807	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1808	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1809	#endif
1810
1811	AssertPtr(pVM);
1812	AssertPtr(pCpu);
1813	AssertPtr(pVCpu);
1814	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1815	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1816	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1817
1818
1819	/*
1820	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1821	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1822	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1823	*/
1824	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1825	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1826	{
1827	++pCpu->uCurrentAsid;
1828	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1829	{
1830	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1831	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1832	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1833	}
1834
1835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1836	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1837	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1838
1839	/*
1840	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1841	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1842	*/
1843	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1844	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1845	HMVMX_SET_TAGGED_TLB_FLUSHED();
1846	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1847	}
1848
1849	/* Check for explicit TLB shootdowns. */
1850	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1851	{
1852	/*
1853	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1854	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1855	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1856	* but not guest-physical mappings.
1857	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1858	*/
1859	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1860	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1861	HMVMX_SET_TAGGED_TLB_FLUSHED();
1862	}
1863
1864	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1865	* where it is commented out. Support individual entry flushing
1866	* someday. */
1867	#if 0
1868	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1869	{
1870	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1871
1872	/*
1873	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1874	* as supported by the CPU.
1875	*/
1876	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1877	{
1878	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1879	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1880	}
1881	else
1882	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1883
1884	HMVMX_SET_TAGGED_TLB_FLUSHED();
1885	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1886	pVCpu->hm.s.TlbShootdown.cPages = 0;
1887	}
1888	#endif
1889
1890	pVCpu->hm.s.fForceTLBFlush = false;
1891
1892	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1893
1894	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1895	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1896	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1897	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1898	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1899	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1900	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1901	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1902
1903	/* Update VMCS with the VPID. */
1904	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1905	AssertRC(rc);
1906
1907	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1908	}
1909
1910
1911	/**
1912	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	* @param pVCpu Pointer to the VMCPU.
1917	* @param pCpu Pointer to the global HM CPU struct.
1918	*
1919	* @remarks Called with interrupts disabled.
1920	*/
1921	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922	{
1923	AssertPtr(pVM);
1924	AssertPtr(pVCpu);
1925	AssertPtr(pCpu);
1926	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1927	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1928
1929	/*
1930	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1931	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1932	*/
1933	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1934	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1935	{
1936	pVCpu->hm.s.fForceTLBFlush = true;
1937	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938	}
1939
1940	/* Check for explicit TLB shootdown flushes. */
1941	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1942	{
1943	pVCpu->hm.s.fForceTLBFlush = true;
1944	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1945	}
1946
1947	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1948	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1949
1950	if (pVCpu->hm.s.fForceTLBFlush)
1951	{
1952	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953	pVCpu->hm.s.fForceTLBFlush = false;
1954	}
1955	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1956	* where it is commented out. Support individual entry flushing
1957	* someday. */
1958	#if 0
1959	else
1960	{
1961	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962	{
1963	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1964	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1965	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1966	}
1967	else
1968	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1969
1970	pVCpu->hm.s.TlbShootdown.cPages = 0;
1971	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972	}
1973	#endif
1974	}
1975
1976
1977	/**
1978	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1979	*
1980	* @returns VBox status code.
1981	* @param pVM Pointer to the VM.
1982	* @param pVCpu Pointer to the VMCPU.
1983	* @param pCpu Pointer to the global HM CPU struct.
1984	*
1985	* @remarks Called with interrupts disabled.
1986	*/
1987	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1988	{
1989	AssertPtr(pVM);
1990	AssertPtr(pVCpu);
1991	AssertPtr(pCpu);
1992	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1993	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1994
1995	/*
1996	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1997	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1998	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1999	*/
2000	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2001	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2002	{
2003	pVCpu->hm.s.fForceTLBFlush = true;
2004	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2005	}
2006
2007	/* Check for explicit TLB shootdown flushes. */
2008	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2009	{
2010	/*
2011	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2012	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2013	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2014	*/
2015	pVCpu->hm.s.fForceTLBFlush = true;
2016	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2017	}
2018
2019	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2020	if (pVCpu->hm.s.fForceTLBFlush)
2021	{
2022	++pCpu->uCurrentAsid;
2023	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2024	{
2025	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2026	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2027	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2028	}
2029
2030	pVCpu->hm.s.fForceTLBFlush = false;
2031	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2032	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2033	if (pCpu->fFlushAsidBeforeUse)
2034	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2035	}
2036	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2037	* where it is commented out. Support individual entry flushing
2038	* someday. */
2039	#if 0
2040	else
2041	{
2042	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2043	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2044	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2045	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2046
2047	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2048	{
2049	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2050	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2051	{
2052	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2053	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2054	}
2055	else
2056	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2057
2058	pVCpu->hm.s.TlbShootdown.cPages = 0;
2059	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060	}
2061	else
2062	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2063	}
2064	#endif
2065
2066	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2067	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2068	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2069	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
2070	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2074	AssertRC(rc);
2075	}
2076
2077
2078	/**
2079	* Flushes the guest TLB entry based on CPU capabilities.
2080	*
2081	* @param pVCpu Pointer to the VMCPU.
2082	* @param pCpu Pointer to the global HM CPU struct.
2083	*/
2084	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2087	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2088	#endif
2089	PVM pVM = pVCpu->CTX_SUFF(pVM);
2090	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2091	{
2092	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2093	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2094	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2095	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2096	default:
2097	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2098	break;
2099	}
2100
2101	/* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2102	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2103
2104	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer pending. It can be set by other EMTs. */
2105	}
2106
2107
2108	/**
2109	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2110	* TLB entries from the host TLB before VM-entry.
2111	*
2112	* @returns VBox status code.
2113	* @param pVM Pointer to the VM.
2114	*/
2115	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2116	{
2117	/*
2118	* Determine optimal flush type for Nested Paging.
2119	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2120	* guest execution (see hmR3InitFinalizeR0()).
2121	*/
2122	if (pVM->hm.s.fNestedPaging)
2123	{
2124	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2125	{
2126	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2127	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2128	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2129	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2130	else
2131	{
2132	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2133	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2134	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2135	}
2136
2137	/* Make sure the write-back cacheable memory type for EPT is supported. */
2138	if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2139	{
2140	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2141	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2142	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2143	}
2144	}
2145	else
2146	{
2147	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2148	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2149	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2150	}
2151	}
2152
2153	/*
2154	* Determine optimal flush type for VPID.
2155	*/
2156	if (pVM->hm.s.vmx.fVpid)
2157	{
2158	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2159	{
2160	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2161	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2162	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2163	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2164	else
2165	{
2166	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2167	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2169	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2170	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2171	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2172	pVM->hm.s.vmx.fVpid = false;
2173	}
2174	}
2175	else
2176	{
2177	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2178	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2179	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2180	pVM->hm.s.vmx.fVpid = false;
2181	}
2182	}
2183
2184	/*
2185	* Setup the handler for flushing tagged-TLBs.
2186	*/
2187	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2188	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2189	else if (pVM->hm.s.fNestedPaging)
2190	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2191	else if (pVM->hm.s.vmx.fVpid)
2192	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2193	else
2194	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Sets up pin-based VM-execution controls in the VMCS.
2201	*
2202	* @returns VBox status code.
2203	* @param pVM Pointer to the VM.
2204	* @param pVCpu Pointer to the VMCPU.
2205	*/
2206	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2207	{
2208	AssertPtr(pVM);
2209	AssertPtr(pVCpu);
2210
2211	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2212	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2213
2214	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2215	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2216	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2217
2218	/* Enable the VMX preemption timer. */
2219	if (pVM->hm.s.vmx.fUsePreemptTimer)
2220	{
2221	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2222	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2223	}
2224
2225	if ((val & zap) != val)
2226	{
2227	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2228	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2229	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2230	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2231	}
2232
2233	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2234	AssertRCReturn(rc, rc);
2235
2236	/* Update VCPU with the currently set pin-based VM-execution controls. */
2237	pVCpu->hm.s.vmx.u32PinCtls = val;
2238	return rc;
2239	}
2240
2241
2242	/**
2243	* Sets up processor-based VM-execution controls in the VMCS.
2244	*
2245	* @returns VBox status code.
2246	* @param pVM Pointer to the VM.
2247	* @param pVMCPU Pointer to the VMCPU.
2248	*/
2249	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2250	{
2251	AssertPtr(pVM);
2252	AssertPtr(pVCpu);
2253
2254	int rc = VERR_INTERNAL_ERROR_5;
2255	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2256	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2257
2258	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2259	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2260	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2261	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2262	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2263	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2264	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2265
2266	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2267	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2268	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2269	{
2270	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2271	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2272	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273	}
2274
2275	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2276	if (!pVM->hm.s.fNestedPaging)
2277	{
2278	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2279	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2280	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2281	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2282	}
2283
2284	/* Use TPR shadowing if supported by the CPU. */
2285	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2286	{
2287	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2288	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2289	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2290	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2291	AssertRCReturn(rc, rc);
2292
2293	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2294	/* CR8 writes causes a VM-exit based on TPR threshold. */
2295	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2296	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2297	}
2298	else
2299	{
2300	/*
2301	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2302	* Set this control only for 64-bit guests.
2303	*/
2304	if (pVM->hm.s.fAllow64BitGuests)
2305	{
2306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2307	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2308	}
2309	}
2310
2311	/* Use MSR-bitmaps if supported by the CPU. */
2312	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2313	{
2314	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2315
2316	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2317	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2319	AssertRCReturn(rc, rc);
2320
2321	/*
2322	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2323	* automatically as dedicated fields in the VMCS.
2324	*/
2325	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2326	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2327	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2328	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2329	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2330
2331	#if HC_ARCH_BITS == 64
2332	/*
2333	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2334	*/
2335	if (pVM->hm.s.fAllow64BitGuests)
2336	{
2337	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2338	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2339	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2340	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2341	}
2342	#endif
2343	}
2344
2345	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2346	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2347	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2348
2349	if ((val & zap) != val)
2350	{
2351	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2352	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2353	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set processor-based VM-execution controls. */
2361	pVCpu->hm.s.vmx.u32ProcCtls = val;
2362
2363	/*
2364	* Secondary processor-based VM-execution controls.
2365	*/
2366	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2367	{
2368	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2373
2374	if (pVM->hm.s.fNestedPaging)
2375	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2376	else
2377	{
2378	/*
2379	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2380	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2381	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2382	*/
2383	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2384	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2385	}
2386
2387	if (pVM->hm.s.vmx.fVpid)
2388	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2389
2390	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2391	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2392
2393	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2394	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2395	* done dynamically. */
2396	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2397	{
2398	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2399	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2400	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2401	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2402	AssertRCReturn(rc, rc);
2403	}
2404
2405	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2406	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2411	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2412	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2413	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2414	}
2415
2416	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2417	AssertRCReturn(rc, rc);
2418
2419	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2420	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2421	}
2422	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2423	{
2424	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2425	"available\n"));
2426	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2427	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2428	}
2429
2430	return VINF_SUCCESS;
2431	}
2432
2433
2434	/**
2435	* Sets up miscellaneous (everything other than Pin & Processor-based
2436	* VM-execution) control fields in the VMCS.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	*/
2442	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2443	{
2444	NOREF(pVM);
2445	AssertPtr(pVM);
2446	AssertPtr(pVCpu);
2447
2448	int rc = VERR_GENERAL_FAILURE;
2449
2450	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2451	#if 0
2452	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2453	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2454	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2455
2456	/*
2457	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2458	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2459	* We thus use the exception bitmap to control it rather than use both.
2460	*/
2461	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2463
2464	/** @todo Explore possibility of using IO-bitmaps. */
2465	/* All IO & IOIO instructions cause VM-exits. */
2466	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2467	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2468
2469	/* Initialize the MSR-bitmap area. */
2470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2473	#endif
2474
2475	/* Setup MSR auto-load/store area. */
2476	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2477	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2479	AssertRCReturn(rc, rc);
2480	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2481	AssertRCReturn(rc, rc);
2482
2483	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2484	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2485	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2486	AssertRCReturn(rc, rc);
2487
2488	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2489	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2490	AssertRCReturn(rc, rc);
2491
2492	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2493	#if 0
2494	/* Setup debug controls */
2495	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2496	AssertRCReturn(rc, rc);
2497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2498	AssertRCReturn(rc, rc);
2499	#endif
2500
2501	return rc;
2502	}
2503
2504
2505	/**
2506	* Sets up the initial exception bitmap in the VMCS based on static conditions
2507	* (i.e. conditions that cannot ever change after starting the VM).
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	*/
2513	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2514	{
2515	AssertPtr(pVM);
2516	AssertPtr(pVCpu);
2517
2518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2519
2520	uint32_t u32XcptBitmap = 0;
2521
2522	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2523	if (!pVM->hm.s.fNestedPaging)
2524	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2525
2526	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2527	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2528	AssertRCReturn(rc, rc);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Sets up the initial guest-state mask. The guest-state mask is consulted
2535	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2536	* for the nested virtualization case (as it would cause a VM-exit).
2537	*
2538	* @param pVCpu Pointer to the VMCPU.
2539	*/
2540	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2541	{
2542	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2543	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2544	return VINF_SUCCESS;
2545	}
2546
2547
2548	/**
2549	* Does per-VM VT-x initialization.
2550	*
2551	* @returns VBox status code.
2552	* @param pVM Pointer to the VM.
2553	*/
2554	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2555	{
2556	LogFlowFunc(("pVM=%p\n", pVM));
2557
2558	int rc = hmR0VmxStructsAlloc(pVM);
2559	if (RT_FAILURE(rc))
2560	{
2561	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2562	return rc;
2563	}
2564
2565	return VINF_SUCCESS;
2566	}
2567
2568
2569	/**
2570	* Does per-VM VT-x termination.
2571	*
2572	* @returns VBox status code.
2573	* @param pVM Pointer to the VM.
2574	*/
2575	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2576	{
2577	LogFlowFunc(("pVM=%p\n", pVM));
2578
2579	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2580	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2581	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2582	#endif
2583	hmR0VmxStructsFree(pVM);
2584	return VINF_SUCCESS;
2585	}
2586
2587
2588	/**
2589	* Sets up the VM for execution under VT-x.
2590	* This function is only called once per-VM during initialization.
2591	*
2592	* @returns VBox status code.
2593	* @param pVM Pointer to the VM.
2594	*/
2595	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2596	{
2597	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2599
2600	LogFlowFunc(("pVM=%p\n", pVM));
2601
2602	/*
2603	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2604	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2605	*/
2606	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2607	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2608	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2609	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2610	{
2611	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2612	return VERR_INTERNAL_ERROR;
2613	}
2614
2615	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2616	/*
2617	* This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2618	* the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2619	*/
2620	if ( pVM->hm.s.fAllow64BitGuests
2621	&& !HMVMX_IS_64BIT_HOST_MODE())
2622	{
2623	LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2624	return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2625	}
2626	#endif
2627
2628	/* Initialize these always, see hmR3InitFinalizeR0().*/
2629	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2630	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2631
2632	/* Setup the tagged-TLB flush handlers. */
2633	int rc = hmR0VmxSetupTaggedTlb(pVM);
2634	if (RT_FAILURE(rc))
2635	{
2636	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2637	return rc;
2638	}
2639
2640	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2641	{
2642	PVMCPU pVCpu = &pVM->aCpus[i];
2643	AssertPtr(pVCpu);
2644	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2645
2646	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2647	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2648
2649	/* Set revision dword at the beginning of the VMCS structure. */
2650	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2651
2652	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2653	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2654	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2655	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2656
2657	/* Load this VMCS as the current VMCS. */
2658	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2659	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2660	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2661
2662	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2663	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2664	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2665
2666	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2667	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2668	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2669
2670	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2671	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2672	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2673
2674	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2675	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2676	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2677
2678	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2679	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2680	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2681
2682	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2683	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2684	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2685	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2686	#endif
2687
2688	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2689	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2690	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2691	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2692
2693	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2694
2695	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2696	}
2697
2698	return VINF_SUCCESS;
2699	}
2700
2701
2702	/**
2703	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2704	* the VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVM Pointer to the VM.
2708	* @param pVCpu Pointer to the VMCPU.
2709	*/
2710	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2711	{
2712	NOREF(pVM); NOREF(pVCpu);
2713
2714	RTCCUINTREG uReg = ASMGetCR0();
2715	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2716	AssertRCReturn(rc, rc);
2717
2718	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2720	if (HMVMX_IS_64BIT_HOST_MODE())
2721	{
2722	uint64_t uRegCR3 = HMR0Get64bitCR3();
2723	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2724	}
2725	else
2726	#endif
2727	{
2728	uReg = ASMGetCR3();
2729	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2730	}
2731	AssertRCReturn(rc, rc);
2732
2733	uReg = ASMGetCR4();
2734	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2735	AssertRCReturn(rc, rc);
2736	return rc;
2737	}
2738
2739
2740	#if HC_ARCH_BITS == 64
2741	/**
2742	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2743	* requirements. See hmR0VmxSaveHostSegmentRegs().
2744	*/
2745	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2746	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2747	{ \
2748	bool fValidSelector = true; \
2749	if ((selValue) & X86_SEL_LDT) \
2750	{ \
2751	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2752	fValidSelector = RT_BOOL(uAttr != ~0U && (uAttr & X86_DESC_P)); \
2753	} \
2754	if (fValidSelector) \
2755	{ \
2756	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2757	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2758	} \
2759	(selValue) = 0; \
2760	}
2761	#endif
2762
2763
2764	/**
2765	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2766	* the host-state area in the VMCS.
2767	*
2768	* @returns VBox status code.
2769	* @param pVM Pointer to the VM.
2770	* @param pVCpu Pointer to the VMCPU.
2771	*/
2772	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2773	{
2774	NOREF(pVM);
2775	int rc = VERR_INTERNAL_ERROR_5;
2776
2777	/*
2778	* Host DS, ES, FS and GS segment registers.
2779	*/
2780	#if HC_ARCH_BITS == 64
2781	RTSEL uSelDS = ASMGetDS();
2782	RTSEL uSelES = ASMGetES();
2783	RTSEL uSelFS = ASMGetFS();
2784	RTSEL uSelGS = ASMGetGS();
2785	#else
2786	RTSEL uSelDS = 0;
2787	RTSEL uSelES = 0;
2788	RTSEL uSelFS = 0;
2789	RTSEL uSelGS = 0;
2790	#endif
2791
2792	/* Recalculate which host-state bits need to be manually restored. */
2793	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2794
2795	/*
2796	* Host CS and SS segment registers.
2797	*/
2798	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2799	RTSEL uSelCS;
2800	RTSEL uSelSS;
2801	if (HMVMX_IS_64BIT_HOST_MODE())
2802	{
2803	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2804	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2805	}
2806	else
2807	{
2808	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2809	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2810	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2811	}
2812	#else
2813	RTSEL uSelCS = ASMGetCS();
2814	RTSEL uSelSS = ASMGetSS();
2815	#endif
2816
2817	/*
2818	* Host TR segment register.
2819	*/
2820	RTSEL uSelTR = ASMGetTR();
2821
2822	#if HC_ARCH_BITS == 64
2823	/*
2824	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2825	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2826	*/
2827	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2828	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2829	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2830	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2831	# undef VMXLOCAL_ADJUST_HOST_SEG
2832	#endif
2833
2834	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2835	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2836	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2837	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2838	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2839	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2840	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2841	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2842	Assert(uSelCS);
2843	Assert(uSelTR);
2844
2845	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2846	#if 0
2847	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2848	Assert(uSelSS != 0);
2849	#endif
2850
2851	/* Write these host selector fields into the host-state area in the VMCS. */
2852	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2853	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2854	#if HC_ARCH_BITS == 64
2855	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2856	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2857	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2858	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2859	#endif
2860	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2861
2862	/*
2863	* Host GDTR and IDTR.
2864	*/
2865	RTGDTR Gdtr;
2866	RT_ZERO(Gdtr);
2867	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2868	if (HMVMX_IS_64BIT_HOST_MODE())
2869	{
2870	X86XDTR64 Gdtr64;
2871	X86XDTR64 Idtr64;
2872	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2873	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2874	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2875
2876	Gdtr.cbGdt = Gdtr64.cb;
2877	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2878	}
2879	else
2880	#endif
2881	{
2882	RTIDTR Idtr;
2883	ASMGetGDTR(&Gdtr);
2884	ASMGetIDTR(&Idtr);
2885	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2886	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2887
2888	#if HC_ARCH_BITS == 64
2889	/*
2890	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2891	* maximum limit (0xffff) on every VM-exit.
2892	*/
2893	if (Gdtr.cbGdt != 0xffff)
2894	{
2895	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2896	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2897	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2898	}
2899
2900	/*
2901	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2902	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2903	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2904	*/
2905	if (Idtr.cbIdt < 0x0fff)
2906	{
2907	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2908	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2909	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2910	}
2911	#endif
2912	}
2913
2914	/*
2915	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2916	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2917	*/
2918	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2919	{
2920	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2921	return VERR_VMX_INVALID_HOST_STATE;
2922	}
2923
2924	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2925	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2926	if (HMVMX_IS_64BIT_HOST_MODE())
2927	{
2928	/* We need the 64-bit TR base for hybrid darwin. */
2929	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2930	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2931	}
2932	else
2933	#endif
2934	{
2935	uintptr_t uTRBase;
2936	#if HC_ARCH_BITS == 64
2937	uTRBase = X86DESC64_BASE(pDesc);
2938
2939	/*
2940	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2941	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2942	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2943	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2944	*
2945	* [1] See Intel spec. 3.5 "System Descriptor Types".
2946	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2947	*/
2948	Assert(pDesc->System.u4Type == 11);
2949	if ( pDesc->System.u16LimitLow != 0x67
2950	\|\| pDesc->System.u4LimitHigh)
2951	{
2952	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
2953	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2954
2955	/* Store the GDTR here as we need it while restoring TR. */
2956	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2957	}
2958	#else
2959	uTRBase = X86DESC_BASE(pDesc);
2960	#endif
2961	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2962	}
2963	AssertRCReturn(rc, rc);
2964
2965	/*
2966	* Host FS base and GS base.
2967	*/
2968	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2969	if (HMVMX_IS_64BIT_HOST_MODE())
2970	{
2971	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2972	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2973	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2974	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2975
2976	# if HC_ARCH_BITS == 64
2977	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2978	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2979	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2980	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2981	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2982	# endif
2983	}
2984	#endif
2985	return rc;
2986	}
2987
2988
2989	/**
2990	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2991	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2992	* the host after every successful VM-exit.
2993	*
2994	* @returns VBox status code.
2995	* @param pVM Pointer to the VM.
2996	* @param pVCpu Pointer to the VMCPU.
2997	*
2998	* @remarks No-long-jump zone!!!
2999	*/
3000	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3001	{
3002	NOREF(pVM);
3003
3004	AssertPtr(pVCpu);
3005	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3006
3007	int rc = VINF_SUCCESS;
3008	#if HC_ARCH_BITS == 64
3009	if (pVM->hm.s.fAllow64BitGuests)
3010	hmR0VmxLazySaveHostMsrs(pVCpu);
3011	#endif
3012
3013	if (pVCpu->hm.s.vmx.cMsrs > 0)
3014	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
3015
3016	/*
3017	* Host Sysenter MSRs.
3018	*/
3019	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3020	AssertRCReturn(rc, rc);
3021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3022	if (HMVMX_IS_64BIT_HOST_MODE())
3023	{
3024	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3025	AssertRCReturn(rc, rc);
3026	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3027	}
3028	else
3029	{
3030	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3031	AssertRCReturn(rc, rc);
3032	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3033	}
3034	#elif HC_ARCH_BITS == 32
3035	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3036	AssertRCReturn(rc, rc);
3037	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3038	#else
3039	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3040	AssertRCReturn(rc, rc);
3041	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3042	#endif
3043	AssertRCReturn(rc, rc);
3044
3045	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
3046	* hmR0VmxSetupExitCtls() !! */
3047	return rc;
3048	}
3049
3050
3051	/**
3052	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3053	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3054	* controls".
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu Pointer to the VMCPU.
3058	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3059	* out-of-sync. Make sure to update the required fields
3060	* before using them.
3061	*
3062	* @remarks No-long-jump zone!!!
3063	*/
3064	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3065	{
3066	int rc = VINF_SUCCESS;
3067	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3068	{
3069	PVM pVM = pVCpu->CTX_SUFF(pVM);
3070	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3071	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3072
3073	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3074	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3075
3076	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3077	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3078	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3079	else
3080	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3081
3082	/*
3083	* The following should -not- be set (since we're not in SMM mode):
3084	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3085	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3086	*/
3087
3088	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3089	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
3090	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
3091
3092	if ((val & zap) != val)
3093	{
3094	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3095	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3096	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3097	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3098	}
3099
3100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3101	AssertRCReturn(rc, rc);
3102
3103	/* Update VCPU with the currently set VM-exit controls. */
3104	pVCpu->hm.s.vmx.u32EntryCtls = val;
3105	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3106	}
3107	return rc;
3108	}
3109
3110
3111	/**
3112	* Sets up the VM-exit controls in the VMCS.
3113	*
3114	* @returns VBox status code.
3115	* @param pVM Pointer to the VM.
3116	* @param pVCpu Pointer to the VMCPU.
3117	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3118	* out-of-sync. Make sure to update the required fields
3119	* before using them.
3120	*
3121	* @remarks requires EFER.
3122	*/
3123	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3124	{
3125	NOREF(pMixedCtx);
3126
3127	int rc = VINF_SUCCESS;
3128	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3129	{
3130	PVM pVM = pVCpu->CTX_SUFF(pVM);
3131	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3132	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3133
3134	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3135	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3136
3137	/*
3138	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3139	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3140	*/
3141	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3142	if (HMVMX_IS_64BIT_HOST_MODE())
3143	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3144	else
3145	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3146	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3147	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3148	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
3149	else
3150	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3151	#endif
3152
3153	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3154	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3155
3156	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3157	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3158	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
3159	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
3160	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
3161
3162	if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3163	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3164
3165	if ((val & zap) != val)
3166	{
3167	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3168	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3169	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3170	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3171	}
3172
3173	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3174	AssertRCReturn(rc, rc);
3175
3176	/* Update VCPU with the currently set VM-exit controls. */
3177	pVCpu->hm.s.vmx.u32ExitCtls = val;
3178	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3179	}
3180	return rc;
3181	}
3182
3183
3184	/**
3185	* Loads the guest APIC and related state.
3186	*
3187	* @returns VBox status code.
3188	* @param pVM Pointer to the VM.
3189	* @param pVCpu Pointer to the VMCPU.
3190	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3191	* out-of-sync. Make sure to update the required fields
3192	* before using them.
3193	*/
3194	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3195	{
3196	NOREF(pMixedCtx);
3197
3198	int rc = VINF_SUCCESS;
3199	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3200	{
3201	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3202	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3203	{
3204	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3205
3206	bool fPendingIntr = false;
3207	uint8_t u8Tpr = 0;
3208	uint8_t u8PendingIntr = 0;
3209	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3210	AssertRCReturn(rc, rc);
3211
3212	/*
3213	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3214	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3215	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3216	* the interrupt when we VM-exit for other reasons.
3217	*/
3218	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3219	uint32_t u32TprThreshold = 0;
3220	if (fPendingIntr)
3221	{
3222	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3223	const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3224	const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3225	if (u8PendingPriority <= u8TprPriority)
3226	u32TprThreshold = u8PendingPriority;
3227	else
3228	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3229	}
3230	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3231
3232	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3233	AssertRCReturn(rc, rc);
3234	}
3235
3236	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3237	}
3238	return rc;
3239	}
3240
3241
3242	/**
3243	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3244	*
3245	* @returns Guest's interruptibility-state.
3246	* @param pVCpu Pointer to the VMCPU.
3247	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3248	* out-of-sync. Make sure to update the required fields
3249	* before using them.
3250	*
3251	* @remarks No-long-jump zone!!!
3252	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3253	*/
3254	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3255	{
3256	/*
3257	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3258	* inhibit interrupts or clear any existing interrupt-inhibition.
3259	*/
3260	uint32_t uIntrState = 0;
3261	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3262	{
3263	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3264	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3265	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3266	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3267	{
3268	/*
3269	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3270	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3271	*/
3272	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3273	}
3274	else if (pMixedCtx->eflags.Bits.u1IF)
3275	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3276	else
3277	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3278	}
3279	return uIntrState;
3280	}
3281
3282
3283	/**
3284	* Loads the guest's interruptibility-state into the guest-state area in the
3285	* VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param uIntrState The interruptibility-state to set.
3290	*/
3291	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3292	{
3293	NOREF(pVCpu);
3294	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3295	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3296	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3297	AssertRCReturn(rc, rc);
3298	return rc;
3299	}
3300
3301
3302	/**
3303	* Loads the guest's RIP into the guest-state area in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu Pointer to the VMCPU.
3307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308	* out-of-sync. Make sure to update the required fields
3309	* before using them.
3310	*
3311	* @remarks No-long-jump zone!!!
3312	*/
3313	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3314	{
3315	int rc = VINF_SUCCESS;
3316	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3317	{
3318	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3319	AssertRCReturn(rc, rc);
3320
3321	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3322	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3323	}
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Loads the guest's RSP into the guest-state area in the VMCS.
3330	*
3331	* @returns VBox status code.
3332	* @param pVCpu Pointer to the VMCPU.
3333	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3334	* out-of-sync. Make sure to update the required fields
3335	* before using them.
3336	*
3337	* @remarks No-long-jump zone!!!
3338	*/
3339	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3340	{
3341	int rc = VINF_SUCCESS;
3342	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3343	{
3344	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3345	AssertRCReturn(rc, rc);
3346
3347	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3348	Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3349	}
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu Pointer to the VMCPU.
3359	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3360	* out-of-sync. Make sure to update the required fields
3361	* before using them.
3362	*
3363	* @remarks No-long-jump zone!!!
3364	*/
3365	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3366	{
3367	int rc = VINF_SUCCESS;
3368	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3369	{
3370	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3371	Let us assert it as such and use 32-bit VMWRITE. */
3372	Assert(!(pMixedCtx->rflags.u64 >> 32));
3373	X86EFLAGS Eflags = pMixedCtx->eflags;
3374	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3375	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3376
3377	/*
3378	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3379	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3380	*/
3381	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3382	{
3383	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3384	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3385	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3386	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3387	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3388	}
3389
3390	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3391	AssertRCReturn(rc, rc);
3392
3393	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3394	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3395	}
3396	return rc;
3397	}
3398
3399
3400	/**
3401	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3402	*
3403	* @returns VBox status code.
3404	* @param pVCpu Pointer to the VMCPU.
3405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3406	* out-of-sync. Make sure to update the required fields
3407	* before using them.
3408	*
3409	* @remarks No-long-jump zone!!!
3410	*/
3411	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3412	{
3413	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3414	AssertRCReturn(rc, rc);
3415	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3416	AssertRCReturn(rc, rc);
3417	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3418	AssertRCReturn(rc, rc);
3419	return rc;
3420	}
3421
3422
3423	/**
3424	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3425	* CR0 is partially shared with the host and we have to consider the FPU bits.
3426	*
3427	* @returns VBox status code.
3428	* @param pVM Pointer to the VM.
3429	* @param pVCpu Pointer to the VMCPU.
3430	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3431	* out-of-sync. Make sure to update the required fields
3432	* before using them.
3433	*
3434	* @remarks No-long-jump zone!!!
3435	*/
3436	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437	{
3438	/*
3439	* Guest CR0.
3440	* Guest FPU.
3441	*/
3442	int rc = VINF_SUCCESS;
3443	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3444	{
3445	Assert(!(pMixedCtx->cr0 >> 32));
3446	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3447	PVM pVM = pVCpu->CTX_SUFF(pVM);
3448
3449	/* The guest's view (read access) of its CR0 is unblemished. */
3450	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3451	AssertRCReturn(rc, rc);
3452	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3453
3454	/* Setup VT-x's view of the guest CR0. */
3455	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3456	if (pVM->hm.s.fNestedPaging)
3457	{
3458	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3459	{
3460	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3461	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3462	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3463	}
3464	else
3465	{
3466	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3467	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3468	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3469	}
3470
3471	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3472	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3473	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3474
3475	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3476	AssertRCReturn(rc, rc);
3477	}
3478	else
3479	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3480
3481	/*
3482	* Guest FPU bits.
3483	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3484	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3485	*/
3486	u32GuestCR0 \|= X86_CR0_NE;
3487	bool fInterceptNM = false;
3488	if (CPUMIsGuestFPUStateActive(pVCpu))
3489	{
3490	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3491	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3492	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3493	}
3494	else
3495	{
3496	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3497	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3498	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3499	}
3500
3501	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3502	bool fInterceptMF = false;
3503	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3504	fInterceptMF = true;
3505
3506	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3507	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3508	{
3509	Assert(PDMVmmDevHeapIsEnabled(pVM));
3510	Assert(pVM->hm.s.vmx.pRealModeTSS);
3511	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3512	fInterceptNM = true;
3513	fInterceptMF = true;
3514	}
3515	else
3516	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3517
3518	if (fInterceptNM)
3519	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3520	else
3521	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3522
3523	if (fInterceptMF)
3524	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3525	else
3526	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3527
3528	/* Additional intercepts for debugging, define these yourself explicitly. */
3529	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3531	\| RT_BIT(X86_XCPT_BP)
3532	\| RT_BIT(X86_XCPT_DB)
3533	\| RT_BIT(X86_XCPT_DE)
3534	\| RT_BIT(X86_XCPT_NM)
3535	\| RT_BIT(X86_XCPT_UD)
3536	\| RT_BIT(X86_XCPT_NP)
3537	\| RT_BIT(X86_XCPT_SS)
3538	\| RT_BIT(X86_XCPT_GP)
3539	\| RT_BIT(X86_XCPT_PF)
3540	\| RT_BIT(X86_XCPT_MF)
3541	;
3542	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3543	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3544	#endif
3545
3546	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3547
3548	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3549	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3550	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3551	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3552	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3553	else
3554	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3555
3556	u32GuestCR0 \|= uSetCR0;
3557	u32GuestCR0 &= uZapCR0;
3558	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3559
3560	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3561	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3562	AssertRCReturn(rc, rc);
3563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3564	AssertRCReturn(rc, rc);
3565	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3566
3567	/*
3568	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3569	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3570	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3571	*/
3572	uint32_t u32CR0Mask = 0;
3573	u32CR0Mask = X86_CR0_PE
3574	\| X86_CR0_NE
3575	\| X86_CR0_WP
3576	\| X86_CR0_PG
3577	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3578	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3579	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3580
3581	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3582	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3583	* and @bugref{6944}. */
3584	#if 0
3585	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3586	u32CR0Mask &= ~X86_CR0_PE;
3587	#endif
3588	if (pVM->hm.s.fNestedPaging)
3589	u32CR0Mask &= ~X86_CR0_WP;
3590
3591	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3592	if (fInterceptNM)
3593	{
3594	u32CR0Mask \|= X86_CR0_TS
3595	\| X86_CR0_MP;
3596	}
3597
3598	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3599	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3600	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3601	AssertRCReturn(rc, rc);
3602	Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3603
3604	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3605	}
3606	return rc;
3607	}
3608
3609
3610	/**
3611	* Loads the guest control registers (CR3, CR4) into the guest-state area
3612	* in the VMCS.
3613	*
3614	* @returns VBox status code.
3615	* @param pVM Pointer to the VM.
3616	* @param pVCpu Pointer to the VMCPU.
3617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3618	* out-of-sync. Make sure to update the required fields
3619	* before using them.
3620	*
3621	* @remarks No-long-jump zone!!!
3622	*/
3623	static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3624	{
3625	int rc = VINF_SUCCESS;
3626	PVM pVM = pVCpu->CTX_SUFF(pVM);
3627
3628	/*
3629	* Guest CR2.
3630	* It's always loaded in the assembler code. Nothing to do here.
3631	*/
3632
3633	/*
3634	* Guest CR3.
3635	*/
3636	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3637	{
3638	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3639	if (pVM->hm.s.fNestedPaging)
3640	{
3641	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3642
3643	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3644	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3645	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3646	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3647
3648	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3649	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3650	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3651
3652	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3653	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3654	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3655	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3656
3657	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3658	AssertRCReturn(rc, rc);
3659	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3660
3661	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3662	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3663	{
3664	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3665	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3666	{
3667	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3668	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3669	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3670	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3671	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3672	}
3673
3674	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3675	have Unrestricted Execution to handle the guest when it's not using paging. */
3676	GCPhysGuestCR3 = pMixedCtx->cr3;
3677	}
3678	else
3679	{
3680	/*
3681	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3682	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3683	* EPT takes care of translating it to host-physical addresses.
3684	*/
3685	RTGCPHYS GCPhys;
3686	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3687	Assert(PDMVmmDevHeapIsEnabled(pVM));
3688
3689	/* We obtain it here every time as the guest could have relocated this PCI region. */
3690	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3691	AssertRCReturn(rc, rc);
3692
3693	GCPhysGuestCR3 = GCPhys;
3694	}
3695
3696	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3697	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3698	}
3699	else
3700	{
3701	/* Non-nested paging case, just use the hypervisor's CR3. */
3702	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3703
3704	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3705	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3706	}
3707	AssertRCReturn(rc, rc);
3708
3709	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3710	}
3711
3712	/*
3713	* Guest CR4.
3714	*/
3715	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3716	{
3717	Assert(!(pMixedCtx->cr4 >> 32));
3718	uint32_t u32GuestCR4 = pMixedCtx->cr4;
3719
3720	/* The guest's view of its CR4 is unblemished. */
3721	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3722	AssertRCReturn(rc, rc);
3723	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3724
3725	/* Setup VT-x's view of the guest CR4. */
3726	/*
3727	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3728	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3729	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3730	*/
3731	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3732	{
3733	Assert(pVM->hm.s.vmx.pRealModeTSS);
3734	Assert(PDMVmmDevHeapIsEnabled(pVM));
3735	u32GuestCR4 &= ~X86_CR4_VME;
3736	}
3737
3738	if (pVM->hm.s.fNestedPaging)
3739	{
3740	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3741	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3742	{
3743	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3744	u32GuestCR4 \|= X86_CR4_PSE;
3745	/* Our identity mapping is a 32-bit page directory. */
3746	u32GuestCR4 &= ~X86_CR4_PAE;
3747	}
3748	/* else use guest CR4.*/
3749	}
3750	else
3751	{
3752	/*
3753	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3754	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3755	*/
3756	switch (pVCpu->hm.s.enmShadowMode)
3757	{
3758	case PGMMODE_REAL: /* Real-mode. */
3759	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3760	case PGMMODE_32_BIT: /* 32-bit paging. */
3761	{
3762	u32GuestCR4 &= ~X86_CR4_PAE;
3763	break;
3764	}
3765
3766	case PGMMODE_PAE: /* PAE paging. */
3767	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3768	{
3769	u32GuestCR4 \|= X86_CR4_PAE;
3770	break;
3771	}
3772
3773	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3774	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3775	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3776	break;
3777	#endif
3778	default:
3779	AssertFailed();
3780	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3781	}
3782	}
3783
3784	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3785	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3786	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3787	u32GuestCR4 \|= uSetCR4;
3788	u32GuestCR4 &= uZapCR4;
3789
3790	/* Write VT-x's view of the guest CR4 into the VMCS. */
3791	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3792	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3793	AssertRCReturn(rc, rc);
3794
3795	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3796	uint32_t u32CR4Mask = 0;
3797	u32CR4Mask = X86_CR4_VME
3798	\| X86_CR4_PAE
3799	\| X86_CR4_PGE
3800	\| X86_CR4_PSE
3801	\| X86_CR4_VMXE;
3802	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3803	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3804	AssertRCReturn(rc, rc);
3805
3806	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3807	}
3808	return rc;
3809	}
3810
3811
3812	/**
3813	* Loads the guest debug registers into the guest-state area in the VMCS.
3814	* This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3815	*
3816	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3817	*
3818	* @returns VBox status code.
3819	* @param pVCpu Pointer to the VMCPU.
3820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3821	* out-of-sync. Make sure to update the required fields
3822	* before using them.
3823	*
3824	* @remarks No-long-jump zone!!!
3825	*/
3826	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3827	{
3828	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3829	return VINF_SUCCESS;
3830
3831	#ifdef VBOX_STRICT
3832	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3833	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3834	{
3835	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3836	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3837	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3838	}
3839	#endif
3840
3841	int rc;
3842	PVM pVM = pVCpu->CTX_SUFF(pVM);
3843	bool fInterceptDB = false;
3844	bool fInterceptMovDRx = false;
3845	if ( pVCpu->hm.s.fSingleInstruction
3846	\|\| DBGFIsStepping(pVCpu))
3847	{
3848	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3849	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3850	{
3851	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3852	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3853	AssertRCReturn(rc, rc);
3854	Assert(fInterceptDB == false);
3855	}
3856	else
3857	{
3858	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3859	pVCpu->hm.s.fClearTrapFlag = true;
3860	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3861	fInterceptDB = true;
3862	}
3863	}
3864
3865	if ( fInterceptDB
3866	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3867	{
3868	/*
3869	* Use the combined guest and host DRx values found in the hypervisor
3870	* register set because the debugger has breakpoints active or someone
3871	* is single stepping on the host side without a monitor trap flag.
3872	*
3873	* Note! DBGF expects a clean DR6 state before executing guest code.
3874	*/
3875	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3876	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3877	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
3878	{
3879	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3880	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3881	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3882	}
3883	else
3884	#endif
3885	if (!CPUMIsHyperDebugStateActive(pVCpu))
3886	{
3887	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3888	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3889	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3890	}
3891
3892	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3893	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3894	AssertRCReturn(rc, rc);
3895
3896	pVCpu->hm.s.fUsingHyperDR7 = true;
3897	fInterceptDB = true;
3898	fInterceptMovDRx = true;
3899	}
3900	else
3901	{
3902	/*
3903	* If the guest has enabled debug registers, we need to load them prior to
3904	* executing guest code so they'll trigger at the right time.
3905	*/
3906	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
3907	{
3908	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3909	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3910	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
3911	{
3912	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3913	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3914	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3915	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3916	}
3917	else
3918	#endif
3919	if (!CPUMIsGuestDebugStateActive(pVCpu))
3920	{
3921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3922	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3924	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3925	}
3926	Assert(!fInterceptDB);
3927	Assert(!fInterceptMovDRx);
3928	}
3929	/*
3930	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3931	* must intercept #DB in order to maintain a correct DR6 guest value.
3932	*/
3933	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3934	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
3935	&& !CPUMIsGuestDebugStateActive(pVCpu))
3936	#else
3937	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3938	#endif
3939	{
3940	fInterceptMovDRx = true;
3941	fInterceptDB = true;
3942	}
3943
3944	/* Update guest DR7. */
3945	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3946	AssertRCReturn(rc, rc);
3947
3948	pVCpu->hm.s.fUsingHyperDR7 = false;
3949	}
3950
3951	/*
3952	* Update the exception bitmap regarding intercepting #DB generated by the guest.
3953	*/
3954	if (fInterceptDB)
3955	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3956	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3957	{
3958	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3959	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3960	#endif
3961	}
3962	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3963	AssertRCReturn(rc, rc);
3964
3965	/*
3966	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3967	*/
3968	if (fInterceptMovDRx)
3969	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3970	else
3971	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3972	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3973	AssertRCReturn(rc, rc);
3974
3975	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3976	return VINF_SUCCESS;
3977	}
3978
3979
3980	#ifdef VBOX_STRICT
3981	/**
3982	* Strict function to validate segment registers.
3983	*
3984	* @remarks ASSUMES CR0 is up to date.
3985	*/
3986	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3987	{
3988	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3989	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3990	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3991	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3992	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3993	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3994	{
3995	/* Protected mode checks */
3996	/* CS */
3997	Assert(pCtx->cs.Attr.n.u1Present);
3998	Assert(!(pCtx->cs.Attr.u & 0xf00));
3999	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4000	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4001	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4002	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4003	\|\| (pCtx->cs.Attr.n.u1Granularity));
4004	/* CS cannot be loaded with NULL in protected mode. */
4005	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4006	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4007	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4008	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4009	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4010	else
4011	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4012	/* SS */
4013	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4014	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4015	if ( !(pCtx->cr0 & X86_CR0_PE)
4016	\|\| pCtx->cs.Attr.n.u4Type == 3)
4017	{
4018	Assert(!pCtx->ss.Attr.n.u2Dpl);
4019	}
4020	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4021	{
4022	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4023	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4024	Assert(pCtx->ss.Attr.n.u1Present);
4025	Assert(!(pCtx->ss.Attr.u & 0xf00));
4026	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4027	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4028	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4029	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4030	\|\| (pCtx->ss.Attr.n.u1Granularity));
4031	}
4032	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4033	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4034	{
4035	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4036	Assert(pCtx->ds.Attr.n.u1Present);
4037	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4038	Assert(!(pCtx->ds.Attr.u & 0xf00));
4039	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4040	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4041	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4042	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4043	\|\| (pCtx->ds.Attr.n.u1Granularity));
4044	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4045	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4046	}
4047	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4048	{
4049	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4050	Assert(pCtx->es.Attr.n.u1Present);
4051	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4052	Assert(!(pCtx->es.Attr.u & 0xf00));
4053	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4054	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4055	\|\| !(pCtx->es.Attr.n.u1Granularity));
4056	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4057	\|\| (pCtx->es.Attr.n.u1Granularity));
4058	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4059	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4060	}
4061	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4062	{
4063	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4064	Assert(pCtx->fs.Attr.n.u1Present);
4065	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4066	Assert(!(pCtx->fs.Attr.u & 0xf00));
4067	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4068	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4069	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4070	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4071	\|\| (pCtx->fs.Attr.n.u1Granularity));
4072	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4073	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4074	}
4075	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4076	{
4077	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4078	Assert(pCtx->gs.Attr.n.u1Present);
4079	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4080	Assert(!(pCtx->gs.Attr.u & 0xf00));
4081	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4082	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4083	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4084	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4085	\|\| (pCtx->gs.Attr.n.u1Granularity));
4086	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4087	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4088	}
4089	/* 64-bit capable CPUs. */
4090	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4091	if (HMVMX_IS_64BIT_HOST_MODE())
4092	{
4093	Assert(!(pCtx->cs.u64Base >> 32));
4094	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4095	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4096	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4097	}
4098	# endif
4099	}
4100	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4101	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4102	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4103	{
4104	/* Real and v86 mode checks. */
4105	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4106	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4107	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4108	{
4109	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4110	}
4111	else
4112	{
4113	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4114	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4115	}
4116
4117	/* CS */
4118	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4119	Assert(pCtx->cs.u32Limit == 0xffff);
4120	Assert(u32CSAttr == 0xf3);
4121	/* SS */
4122	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4123	Assert(pCtx->ss.u32Limit == 0xffff);
4124	Assert(u32SSAttr == 0xf3);
4125	/* DS */
4126	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4127	Assert(pCtx->ds.u32Limit == 0xffff);
4128	Assert(u32DSAttr == 0xf3);
4129	/* ES */
4130	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4131	Assert(pCtx->es.u32Limit == 0xffff);
4132	Assert(u32ESAttr == 0xf3);
4133	/* FS */
4134	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4135	Assert(pCtx->fs.u32Limit == 0xffff);
4136	Assert(u32FSAttr == 0xf3);
4137	/* GS */
4138	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4139	Assert(pCtx->gs.u32Limit == 0xffff);
4140	Assert(u32GSAttr == 0xf3);
4141	/* 64-bit capable CPUs. */
4142	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4143	if (HMVMX_IS_64BIT_HOST_MODE())
4144	{
4145	Assert(!(pCtx->cs.u64Base >> 32));
4146	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4147	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4148	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4149	}
4150	# endif
4151	}
4152	}
4153	#endif /* VBOX_STRICT */
4154
4155
4156	/**
4157	* Writes a guest segment register into the guest-state area in the VMCS.
4158	*
4159	* @returns VBox status code.
4160	* @param pVCpu Pointer to the VMCPU.
4161	* @param idxSel Index of the selector in the VMCS.
4162	* @param idxLimit Index of the segment limit in the VMCS.
4163	* @param idxBase Index of the segment base in the VMCS.
4164	* @param idxAccess Index of the access rights of the segment in the VMCS.
4165	* @param pSelReg Pointer to the segment selector.
4166	*
4167	* @remarks No-long-jump zone!!!
4168	*/
4169	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4170	uint32_t idxAccess, PCPUMSELREG pSelReg)
4171	{
4172	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4173	AssertRCReturn(rc, rc);
4174	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4175	AssertRCReturn(rc, rc);
4176	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4177	AssertRCReturn(rc, rc);
4178
4179	uint32_t u32Access = pSelReg->Attr.u;
4180	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4181	{
4182	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4183	u32Access = 0xf3;
4184	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4185	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4186	}
4187	else
4188	{
4189	/*
4190	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4191	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4192	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4193	* loaded in protected-mode have their attribute as 0.
4194	*/
4195	if (!u32Access)
4196	u32Access = X86DESCATTR_UNUSABLE;
4197	}
4198
4199	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4200	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4201	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4202
4203	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4204	AssertRCReturn(rc, rc);
4205	return rc;
4206	}
4207
4208
4209	/**
4210	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4211	* into the guest-state area in the VMCS.
4212	*
4213	* @returns VBox status code.
4214	* @param pVM Pointer to the VM.
4215	* @param pVCPU Pointer to the VMCPU.
4216	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4217	* out-of-sync. Make sure to update the required fields
4218	* before using them.
4219	*
4220	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4224	{
4225	int rc = VERR_INTERNAL_ERROR_5;
4226	PVM pVM = pVCpu->CTX_SUFF(pVM);
4227
4228	/*
4229	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4230	*/
4231	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4232	{
4233	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4234	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4235	{
4236	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4237	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4238	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4239	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4240	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4241	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4242	}
4243
4244	#ifdef VBOX_WITH_REM
4245	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4246	{
4247	Assert(pVM->hm.s.vmx.pRealModeTSS);
4248	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4249	if ( pVCpu->hm.s.vmx.fWasInRealMode
4250	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4251	{
4252	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4253	in real-mode (e.g. OpenBSD 4.0) */
4254	REMFlushTBs(pVM);
4255	Log4(("Load: Switch to protected mode detected!\n"));
4256	pVCpu->hm.s.vmx.fWasInRealMode = false;
4257	}
4258	}
4259	#endif
4260	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4261	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4262	AssertRCReturn(rc, rc);
4263	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4264	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4265	AssertRCReturn(rc, rc);
4266	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4267	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4268	AssertRCReturn(rc, rc);
4269	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4270	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4271	AssertRCReturn(rc, rc);
4272	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4273	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4274	AssertRCReturn(rc, rc);
4275	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4276	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4277	AssertRCReturn(rc, rc);
4278
4279	#ifdef VBOX_STRICT
4280	/* Validate. */
4281	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4282	#endif
4283
4284	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4285	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4286	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4287	}
4288
4289	/*
4290	* Guest TR.
4291	*/
4292	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4293	{
4294	/*
4295	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4296	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4297	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4298	*/
4299	uint16_t u16Sel = 0;
4300	uint32_t u32Limit = 0;
4301	uint64_t u64Base = 0;
4302	uint32_t u32AccessRights = 0;
4303
4304	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4305	{
4306	u16Sel = pMixedCtx->tr.Sel;
4307	u32Limit = pMixedCtx->tr.u32Limit;
4308	u64Base = pMixedCtx->tr.u64Base;
4309	u32AccessRights = pMixedCtx->tr.Attr.u;
4310	}
4311	else
4312	{
4313	Assert(pVM->hm.s.vmx.pRealModeTSS);
4314	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4315
4316	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4317	RTGCPHYS GCPhys;
4318	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4319	AssertRCReturn(rc, rc);
4320
4321	X86DESCATTR DescAttr;
4322	DescAttr.u = 0;
4323	DescAttr.n.u1Present = 1;
4324	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4325
4326	u16Sel = 0;
4327	u32Limit = HM_VTX_TSS_SIZE;
4328	u64Base = GCPhys; /* in real-mode phys = virt. */
4329	u32AccessRights = DescAttr.u;
4330	}
4331
4332	/* Validate. */
4333	Assert(!(u16Sel & RT_BIT(2)));
4334	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4335	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4336	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4337	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4338	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4339	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4340	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4341	Assert( (u32Limit & 0xfff) == 0xfff
4342	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4343	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4344	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4345
4346	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4347	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4348	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4349	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4350
4351	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4352	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4353	}
4354
4355	/*
4356	* Guest GDTR.
4357	*/
4358	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4359	{
4360	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4361	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4362
4363	/* Validate. */
4364	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4365
4366	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4367	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4368	}
4369
4370	/*
4371	* Guest LDTR.
4372	*/
4373	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4374	{
4375	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4376	uint32_t u32Access = 0;
4377	if (!pMixedCtx->ldtr.Attr.u)
4378	u32Access = X86DESCATTR_UNUSABLE;
4379	else
4380	u32Access = pMixedCtx->ldtr.Attr.u;
4381
4382	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4383	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4384	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4385	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4386
4387	/* Validate. */
4388	if (!(u32Access & X86DESCATTR_UNUSABLE))
4389	{
4390	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4391	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4392	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4393	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4394	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4395	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4396	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4397	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4398	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4399	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4400	}
4401
4402	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4403	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4404	}
4405
4406	/*
4407	* Guest IDTR.
4408	*/
4409	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4410	{
4411	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4412	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4413
4414	/* Validate. */
4415	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4416
4417	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4418	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4419	}
4420
4421	return VINF_SUCCESS;
4422	}
4423
4424
4425	/**
4426	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4427	* areas. These MSRs will automatically be loaded to the host CPU on every
4428	* successful VM entry and stored from the host CPU on every successful VM-exit.
4429	*
4430	* This also creates/updates MSR slots for the host MSRs. The actual host
4431	* MSR values are -not- updated here for performance reasons. See
4432	* hmR0VmxSaveHostMsrs().
4433	*
4434	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	*
4442	* @remarks No-long-jump zone!!!
4443	*/
4444	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4445	{
4446	AssertPtr(pVCpu);
4447	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4448
4449	/*
4450	* MSRs that we use the auto-load/store MSR area in the VMCS.
4451	*/
4452	PVM pVM = pVCpu->CTX_SUFF(pVM);
4453	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4454	{
4455	#if HC_ARCH_BITS == 32 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4456	if (pVM->hm.s.fAllow64BitGuests)
4457	{
4458	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4459	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4460	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4461	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4462	# ifdef DEBUG
4463	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4464	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4465	Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4466	# endif
4467	}
4468	#endif
4469	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4470	}
4471
4472	/*
4473	* Guest Sysenter MSRs.
4474	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4475	* VM-exits on WRMSRs for these MSRs.
4476	*/
4477	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4478	{
4479	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4480	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4481	}
4482
4483	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4484	{
4485	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4486	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4487	}
4488
4489	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4490	{
4491	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4492	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4493	}
4494
4495	return VINF_SUCCESS;
4496	}
4497
4498
4499	/**
4500	* Loads the guest activity state into the guest-state area in the VMCS.
4501	*
4502	* @returns VBox status code.
4503	* @param pVCpu Pointer to the VMCPU.
4504	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4505	* out-of-sync. Make sure to update the required fields
4506	* before using them.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4511	{
4512	NOREF(pCtx);
4513	/** @todo See if we can make use of other states, e.g.
4514	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4515	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4516	{
4517	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4518	AssertRCReturn(rc, rc);
4519
4520	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4521	}
4522	return VINF_SUCCESS;
4523	}
4524
4525
4526	/**
4527	* Sets up the appropriate function to run guest code.
4528	*
4529	* @returns VBox status code.
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4532	* out-of-sync. Make sure to update the required fields
4533	* before using them.
4534	*
4535	* @remarks No-long-jump zone!!!
4536	*/
4537	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4538	{
4539	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4540	{
4541	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4542	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4543	#endif
4544	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4545	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4546	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4547	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4548	{
4549	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4550	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4551	}
4552	#else
4553	/* 64-bit host or hybrid host. */
4554	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4555	#endif
4556	}
4557	else
4558	{
4559	/* Guest is not in long mode, use the 32-bit handler. */
4560	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4561	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4562	{
4563	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4564	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_EXIT_CTLS \| HM_CHANGED_VMX_ENTRY_CTLS);
4565	}
4566	#else
4567	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4568	#endif
4569	}
4570	Assert(pVCpu->hm.s.vmx.pfnStartVM);
4571	return VINF_SUCCESS;
4572	}
4573
4574
4575	/**
4576	* Wrapper for running the guest code in VT-x.
4577	*
4578	* @returns VBox strict status code.
4579	* @param pVM Pointer to the VM.
4580	* @param pVCpu Pointer to the VMCPU.
4581	* @param pCtx Pointer to the guest-CPU context.
4582	*
4583	* @remarks No-long-jump zone!!!
4584	*/
4585	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4586	{
4587	/*
4588	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4589	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4590	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4591	*/
4592	const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4593	/** @todo Add stats for resume vs launch. */
4594	#ifdef VBOX_WITH_KERNEL_USING_XMM
4595	return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4596	#else
4597	return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4598	#endif
4599	}
4600
4601
4602	/**
4603	* Reports world-switch error and dumps some useful debug info.
4604	*
4605	* @param pVM Pointer to the VM.
4606	* @param pVCpu Pointer to the VMCPU.
4607	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4608	* @param pCtx Pointer to the guest-CPU context.
4609	* @param pVmxTransient Pointer to the VMX transient structure (only
4610	* exitReason updated).
4611	*/
4612	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4613	{
4614	Assert(pVM);
4615	Assert(pVCpu);
4616	Assert(pCtx);
4617	Assert(pVmxTransient);
4618	HMVMX_ASSERT_PREEMPT_SAFE();
4619
4620	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4621	switch (rcVMRun)
4622	{
4623	case VERR_VMX_INVALID_VMXON_PTR:
4624	AssertFailed();
4625	break;
4626	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4627	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4628	{
4629	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4630	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4631	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4632	AssertRC(rc);
4633
4634	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4635	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4636	Cannot do it here as we may have been long preempted. */
4637
4638	#ifdef VBOX_STRICT
4639	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4640	pVmxTransient->uExitReason));
4641	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4642	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4643	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4644	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4645	else
4646	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4647	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4648	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4649
4650	/* VMX control bits. */
4651	uint32_t u32Val;
4652	uint64_t u64Val;
4653	HMVMXHCUINTREG uHCReg;
4654	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4655	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4656	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4657	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4658	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4659	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4660	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4661	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4662	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4663	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4664	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4665	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4666	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4667	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4668	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4669	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4670	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4671	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4672	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4673	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4674	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4675	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4676	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4677	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4678	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4679	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4680	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4681	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4683	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4684	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4685	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4686	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4687	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4688	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4689	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4690	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4691	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4692	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4693	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4694	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4695	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4696
4697	/* Guest bits. */
4698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4699	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4700	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4701	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4702	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4703	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4704	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4705	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4706
4707	/* Host bits. */
4708	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4709	Log4(("Host CR0 %#RHr\n", uHCReg));
4710	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4711	Log4(("Host CR3 %#RHr\n", uHCReg));
4712	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4713	Log4(("Host CR4 %#RHr\n", uHCReg));
4714
4715	RTGDTR HostGdtr;
4716	PCX86DESCHC pDesc;
4717	ASMGetGDTR(&HostGdtr);
4718	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4719	Log4(("Host CS %#08x\n", u32Val));
4720	if (u32Val < HostGdtr.cbGdt)
4721	{
4722	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4723	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4724	}
4725
4726	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4727	Log4(("Host DS %#08x\n", u32Val));
4728	if (u32Val < HostGdtr.cbGdt)
4729	{
4730	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4731	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4732	}
4733
4734	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4735	Log4(("Host ES %#08x\n", u32Val));
4736	if (u32Val < HostGdtr.cbGdt)
4737	{
4738	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4739	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4740	}
4741
4742	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4743	Log4(("Host FS %#08x\n", u32Val));
4744	if (u32Val < HostGdtr.cbGdt)
4745	{
4746	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4747	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4748	}
4749
4750	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4751	Log4(("Host GS %#08x\n", u32Val));
4752	if (u32Val < HostGdtr.cbGdt)
4753	{
4754	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4755	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4756	}
4757
4758	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4759	Log4(("Host SS %#08x\n", u32Val));
4760	if (u32Val < HostGdtr.cbGdt)
4761	{
4762	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4763	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4764	}
4765
4766	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4767	Log4(("Host TR %#08x\n", u32Val));
4768	if (u32Val < HostGdtr.cbGdt)
4769	{
4770	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4771	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4772	}
4773
4774	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4775	Log4(("Host TR Base %#RHv\n", uHCReg));
4776	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4777	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4778	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4779	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4780	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4781	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4782	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4783	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4784	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4785	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4786	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4787	Log4(("Host RSP %#RHv\n", uHCReg));
4788	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4789	Log4(("Host RIP %#RHv\n", uHCReg));
4790	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4791	if (HMVMX_IS_64BIT_HOST_MODE())
4792	{
4793	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4794	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4795	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4796	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4797	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4798	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4799	}
4800	# endif
4801	#endif /* VBOX_STRICT */
4802	break;
4803	}
4804
4805	default:
4806	/* Impossible */
4807	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4808	break;
4809	}
4810	NOREF(pVM); NOREF(pCtx);
4811	}
4812
4813
4814	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4815	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4816	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4817	#endif
4818	#ifdef VBOX_STRICT
4819	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4820	{
4821	switch (idxField)
4822	{
4823	case VMX_VMCS_GUEST_RIP:
4824	case VMX_VMCS_GUEST_RSP:
4825	case VMX_VMCS_GUEST_SYSENTER_EIP:
4826	case VMX_VMCS_GUEST_SYSENTER_ESP:
4827	case VMX_VMCS_GUEST_GDTR_BASE:
4828	case VMX_VMCS_GUEST_IDTR_BASE:
4829	case VMX_VMCS_GUEST_CS_BASE:
4830	case VMX_VMCS_GUEST_DS_BASE:
4831	case VMX_VMCS_GUEST_ES_BASE:
4832	case VMX_VMCS_GUEST_FS_BASE:
4833	case VMX_VMCS_GUEST_GS_BASE:
4834	case VMX_VMCS_GUEST_SS_BASE:
4835	case VMX_VMCS_GUEST_LDTR_BASE:
4836	case VMX_VMCS_GUEST_TR_BASE:
4837	case VMX_VMCS_GUEST_CR3:
4838	return true;
4839	}
4840	return false;
4841	}
4842
4843	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4844	{
4845	switch (idxField)
4846	{
4847	/* Read-only fields. */
4848	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4849	return true;
4850	}
4851	/* Remaining readable fields should also be writable. */
4852	return hmR0VmxIsValidWriteField(idxField);
4853	}
4854	#endif /* VBOX_STRICT */
4855
4856
4857	/**
4858	* Executes the specified handler in 64-bit mode.
4859	*
4860	* @returns VBox status code.
4861	* @param pVM Pointer to the VM.
4862	* @param pVCpu Pointer to the VMCPU.
4863	* @param pCtx Pointer to the guest CPU context.
4864	* @param enmOp The operation to perform.
4865	* @param cbParam Number of parameters.
4866	* @param paParam Array of 32-bit parameters.
4867	*/
4868	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4869	uint32_t *paParam)
4870	{
4871	int rc, rc2;
4872	PHMGLOBALCPUINFO pCpu;
4873	RTHCPHYS HCPhysCpuPage;
4874	RTCCUINTREG uOldEflags;
4875
4876	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4877	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4878	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4879	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4880
4881	#ifdef VBOX_STRICT
4882	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4883	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4884
4885	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4886	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4887	#endif
4888
4889	/* Disable interrupts. */
4890	uOldEflags = ASMIntDisableFlags();
4891
4892	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4893	RTCPUID idHostCpu = RTMpCpuId();
4894	CPUMR0SetLApic(pVCpu, idHostCpu);
4895	#endif
4896
4897	pCpu = HMR0GetCurrentCpu();
4898	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4899
4900	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4901	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4902
4903	/* Leave VMX Root Mode. */
4904	VMXDisable();
4905
4906	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4907
4908	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4909	CPUMSetHyperEIP(pVCpu, enmOp);
4910	for (int i = (int)cbParam - 1; i >= 0; i--)
4911	CPUMPushHyper(pVCpu, paParam[i]);
4912
4913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4914
4915	/* Call the switcher. */
4916	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4917	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4918
4919	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4920	/* Make sure the VMX instructions don't cause #UD faults. */
4921	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4922
4923	/* Re-enter VMX Root Mode */
4924	rc2 = VMXEnable(HCPhysCpuPage);
4925	if (RT_FAILURE(rc2))
4926	{
4927	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4928	ASMSetFlags(uOldEflags);
4929	return rc2;
4930	}
4931
4932	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4933	AssertRC(rc2);
4934	Assert(!(ASMGetFlags() & X86_EFL_IF));
4935	ASMSetFlags(uOldEflags);
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
4942	* supporting 64-bit guests.
4943	*
4944	* @returns VBox status code.
4945	* @param fResume Whether to VMLAUNCH or VMRESUME.
4946	* @param pCtx Pointer to the guest-CPU context.
4947	* @param pCache Pointer to the VMCS cache.
4948	* @param pVM Pointer to the VM.
4949	* @param pVCpu Pointer to the VMCPU.
4950	*/
4951	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4952	{
4953	uint32_t aParam[6];
4954	PHMGLOBALCPUINFO pCpu = NULL;
4955	RTHCPHYS HCPhysCpuPage = 0;
4956	int rc = VERR_INTERNAL_ERROR_5;
4957
4958	pCpu = HMR0GetCurrentCpu();
4959	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4960
4961	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4962	pCache->uPos = 1;
4963	pCache->interPD = PGMGetInterPaeCR3(pVM);
4964	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4965	#endif
4966
4967	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4968	pCache->TestIn.HCPhysCpuPage = 0;
4969	pCache->TestIn.HCPhysVmcs = 0;
4970	pCache->TestIn.pCache = 0;
4971	pCache->TestOut.HCPhysVmcs = 0;
4972	pCache->TestOut.pCache = 0;
4973	pCache->TestOut.pCtx = 0;
4974	pCache->TestOut.eflags = 0;
4975	#endif
4976
4977	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4978	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4979	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4980	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4981	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4982	aParam[5] = 0;
4983
4984	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4985	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4986	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4987	#endif
4988	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4989
4990	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4991	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4992	Assert(pCtx->dr[4] == 10);
4993	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4994	#endif
4995
4996	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4997	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4998	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4999	pVCpu->hm.s.vmx.HCPhysVmcs));
5000	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5001	pCache->TestOut.HCPhysVmcs));
5002	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5003	pCache->TestOut.pCache));
5004	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5005	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5006	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5007	pCache->TestOut.pCtx));
5008	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5009	#endif
5010	return rc;
5011	}
5012
5013
5014	/**
5015	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5016	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5017	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5018	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5019	*
5020	* @returns VBox status code.
5021	* @param pVM Pointer to the VM.
5022	* @param pVCpu Pointer to the VMCPU.
5023	*/
5024	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5025	{
5026	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5027	{ \
5028	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5029	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5030	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5031	++cReadFields; \
5032	}
5033
5034	AssertPtr(pVM);
5035	AssertPtr(pVCpu);
5036	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5037	uint32_t cReadFields = 0;
5038
5039	/*
5040	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5041	* and serve to indicate exceptions to the rules.
5042	*/
5043
5044	/* Guest-natural selector base fields. */
5045	#if 0
5046	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5047	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5048	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5049	#endif
5050	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5051	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5052	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5053	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5054	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5055	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5056	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5057	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5058	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5059	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5060	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5061	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5062	#if 0
5063	/* Unused natural width guest-state fields. */
5064	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5065	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5066	#endif
5067	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5068	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5069
5070	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5071	#if 0
5072	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5073	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5074	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5075	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5076	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5077	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5078	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5079	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5080	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5081	#endif
5082
5083	/* Natural width guest-state fields. */
5084	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5085	#if 0
5086	/* Currently unused field. */
5087	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5088	#endif
5089
5090	if (pVM->hm.s.fNestedPaging)
5091	{
5092	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5093	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5094	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5095	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5096	}
5097	else
5098	{
5099	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5100	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5101	}
5102
5103	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5104	return VINF_SUCCESS;
5105	}
5106
5107
5108	/**
5109	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5110	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5111	* darwin, running 64-bit guests).
5112	*
5113	* @returns VBox status code.
5114	* @param pVCpu Pointer to the VMCPU.
5115	* @param idxField The VMCS field encoding.
5116	* @param u64Val 16, 32 or 64-bit value.
5117	*/
5118	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5119	{
5120	int rc;
5121	switch (idxField)
5122	{
5123	/*
5124	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5125	*/
5126	/* 64-bit Control fields. */
5127	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5128	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5129	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5130	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5131	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5132	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5133	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5134	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5135	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5136	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5137	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5138	case VMX_VMCS64_CTRL_EPTP_FULL:
5139	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5140	/* 64-bit Guest-state fields. */
5141	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5142	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5143	case VMX_VMCS64_GUEST_PAT_FULL:
5144	case VMX_VMCS64_GUEST_EFER_FULL:
5145	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5146	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5147	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5148	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5149	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5150	/* 64-bit Host-state fields. */
5151	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5152	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5153	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5154	{
5155	rc = VMXWriteVmcs32(idxField, u64Val);
5156	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5157	break;
5158	}
5159
5160	/*
5161	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5162	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5163	*/
5164	/* Natural-width Guest-state fields. */
5165	case VMX_VMCS_GUEST_CR3:
5166	case VMX_VMCS_GUEST_ES_BASE:
5167	case VMX_VMCS_GUEST_CS_BASE:
5168	case VMX_VMCS_GUEST_SS_BASE:
5169	case VMX_VMCS_GUEST_DS_BASE:
5170	case VMX_VMCS_GUEST_FS_BASE:
5171	case VMX_VMCS_GUEST_GS_BASE:
5172	case VMX_VMCS_GUEST_LDTR_BASE:
5173	case VMX_VMCS_GUEST_TR_BASE:
5174	case VMX_VMCS_GUEST_GDTR_BASE:
5175	case VMX_VMCS_GUEST_IDTR_BASE:
5176	case VMX_VMCS_GUEST_RSP:
5177	case VMX_VMCS_GUEST_RIP:
5178	case VMX_VMCS_GUEST_SYSENTER_ESP:
5179	case VMX_VMCS_GUEST_SYSENTER_EIP:
5180	{
5181	if (!(u64Val >> 32))
5182	{
5183	/* If this field is 64-bit, VT-x will zero out the top bits. */
5184	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5185	}
5186	else
5187	{
5188	/* Assert that only the 32->64 switcher case should ever come here. */
5189	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5190	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5191	}
5192	break;
5193	}
5194
5195	default:
5196	{
5197	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5198	rc = VERR_INVALID_PARAMETER;
5199	break;
5200	}
5201	}
5202	AssertRCReturn(rc, rc);
5203	return rc;
5204	}
5205
5206
5207	/**
5208	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5209	* hosts (except darwin) for 64-bit guests.
5210	*
5211	* @param pVCpu Pointer to the VMCPU.
5212	* @param idxField The VMCS field encoding.
5213	* @param u64Val 16, 32 or 64-bit value.
5214	*/
5215	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5216	{
5217	AssertPtr(pVCpu);
5218	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5219
5220	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5221	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5222
5223	/* Make sure there are no duplicates. */
5224	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5225	{
5226	if (pCache->Write.aField[i] == idxField)
5227	{
5228	pCache->Write.aFieldVal[i] = u64Val;
5229	return VINF_SUCCESS;
5230	}
5231	}
5232
5233	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5234	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5235	pCache->Write.cValidEntries++;
5236	return VINF_SUCCESS;
5237	}
5238
5239	/* Enable later when the assembly code uses these as callbacks. */
5240	#if 0
5241	/*
5242	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5243	*
5244	* @param pVCpu Pointer to the VMCPU.
5245	* @param pCache Pointer to the VMCS cache.
5246	*
5247	* @remarks No-long-jump zone!!!
5248	*/
5249	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5250	{
5251	AssertPtr(pCache);
5252	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5253	{
5254	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5255	AssertRC(rc);
5256	}
5257	pCache->Write.cValidEntries = 0;
5258	}
5259
5260
5261	/**
5262	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
5263	*
5264	* @param pVCpu Pointer to the VMCPU.
5265	* @param pCache Pointer to the VMCS cache.
5266	*
5267	* @remarks No-long-jump zone!!!
5268	*/
5269	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5270	{
5271	AssertPtr(pCache);
5272	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5273	{
5274	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5275	AssertRC(rc);
5276	}
5277	}
5278	#endif
5279	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5280
5281
5282	/**
5283	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5284	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5285	* timer.
5286	*
5287	* @returns VBox status code.
5288	* @param pVCpu Pointer to the VMCPU.
5289	*
5290	* @remarks No-long-jump zone!!!
5291	*/
5292	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5293	{
5294	int rc = VERR_INTERNAL_ERROR_5;
5295	bool fOffsettedTsc = false;
5296	PVM pVM = pVCpu->CTX_SUFF(pVM);
5297	if (pVM->hm.s.vmx.fUsePreemptTimer)
5298	{
5299	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5300
5301	/* Make sure the returned values have sane upper and lower boundaries. */
5302	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5303	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5304	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5305	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5306
5307	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5308	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5309	}
5310	else
5311	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5312
5313	if (fOffsettedTsc)
5314	{
5315	uint64_t u64CurTSC = ASMReadTSC();
5316	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5317	{
5318	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5319	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5320
5321	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5322	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5323	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5324	}
5325	else
5326	{
5327	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5328	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5329	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5331	}
5332	}
5333	else
5334	{
5335	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5336	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5338	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5339	}
5340	}
5341
5342
5343	/**
5344	* Determines if an exception is a contributory exception. Contributory
5345	* exceptions are ones which can cause double-faults. Page-fault is
5346	* intentionally not included here as it's a conditional contributory exception.
5347	*
5348	* @returns true if the exception is contributory, false otherwise.
5349	* @param uVector The exception vector.
5350	*/
5351	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5352	{
5353	switch (uVector)
5354	{
5355	case X86_XCPT_GP:
5356	case X86_XCPT_SS:
5357	case X86_XCPT_NP:
5358	case X86_XCPT_TS:
5359	case X86_XCPT_DE:
5360	return true;
5361	default:
5362	break;
5363	}
5364	return false;
5365	}
5366
5367
5368	/**
5369	* Sets an event as a pending event to be injected into the guest.
5370	*
5371	* @param pVCpu Pointer to the VMCPU.
5372	* @param u32IntInfo The VM-entry interruption-information field.
5373	* @param cbInstr The VM-entry instruction length in bytes (for software
5374	* interrupts, exceptions and privileged software
5375	* exceptions).
5376	* @param u32ErrCode The VM-entry exception error code.
5377	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5378	* page-fault.
5379	*
5380	* @remarks Statistics counter assumes this is a guest event being injected or
5381	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5382	* always incremented.
5383	*/
5384	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5385	RTGCUINTPTR GCPtrFaultAddress)
5386	{
5387	Assert(!pVCpu->hm.s.Event.fPending);
5388	pVCpu->hm.s.Event.fPending = true;
5389	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5390	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5391	pVCpu->hm.s.Event.cbInstr = cbInstr;
5392	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5393
5394	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5395	}
5396
5397
5398	/**
5399	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5400	*
5401	* @param pVCpu Pointer to the VMCPU.
5402	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5403	* out-of-sync. Make sure to update the required fields
5404	* before using them.
5405	*/
5406	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5407	{
5408	NOREF(pMixedCtx);
5409	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5410	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5411	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5412	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5413	}
5414
5415
5416	/**
5417	* Handle a condition that occurred while delivering an event through the guest
5418	* IDT.
5419	*
5420	* @returns VBox status code (informational error codes included).
5421	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5422	* @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5423	* continue execution of the guest which will delivery the #DF.
5424	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5425	*
5426	* @param pVCpu Pointer to the VMCPU.
5427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5428	* out-of-sync. Make sure to update the required fields
5429	* before using them.
5430	* @param pVmxTransient Pointer to the VMX transient structure.
5431	*
5432	* @remarks No-long-jump zone!!!
5433	*/
5434	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5435	{
5436	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5437	AssertRCReturn(rc, rc);
5438	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5439	{
5440	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5441	AssertRCReturn(rc, rc);
5442
5443	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5444	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5445	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5446
5447	typedef enum
5448	{
5449	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5450	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5451	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5452	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5453	} VMXREFLECTXCPT;
5454
5455	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5456	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5457	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5458	{
5459	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5460	{
5461	enmReflect = VMXREFLECTXCPT_XCPT;
5462	#ifdef VBOX_STRICT
5463	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5464	&& uExitVector == X86_XCPT_PF)
5465	{
5466	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5467	}
5468	#endif
5469	if ( uExitVector == X86_XCPT_PF
5470	&& uIdtVector == X86_XCPT_PF)
5471	{
5472	pVmxTransient->fVectoringPF = true;
5473	Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5474	}
5475	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5476	&& hmR0VmxIsContributoryXcpt(uExitVector)
5477	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
5478	\|\| uIdtVector == X86_XCPT_PF))
5479	{
5480	enmReflect = VMXREFLECTXCPT_DF;
5481	}
5482	else if (uIdtVector == X86_XCPT_DF)
5483	enmReflect = VMXREFLECTXCPT_TF;
5484	}
5485	else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5486	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5487	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5488	{
5489	/*
5490	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5491	* (whatever they are) as they reoccur when restarting the instruction.
5492	*/
5493	enmReflect = VMXREFLECTXCPT_XCPT;
5494	}
5495	}
5496	else
5497	{
5498	/*
5499	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5500	* interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5501	* original exception to the guest after handling the VM-exit.
5502	*/
5503	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5504	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5505	\|\| uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5506	{
5507	enmReflect = VMXREFLECTXCPT_XCPT;
5508	}
5509	}
5510
5511	switch (enmReflect)
5512	{
5513	case VMXREFLECTXCPT_XCPT:
5514	{
5515	Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5516	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5517	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5518
5519	uint32_t u32ErrCode = 0;
5520	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5521	{
5522	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5523	AssertRCReturn(rc, rc);
5524	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5525	}
5526
5527	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5528	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5529	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5530	rc = VINF_SUCCESS;
5531	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5532	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5533
5534	break;
5535	}
5536
5537	case VMXREFLECTXCPT_DF:
5538	{
5539	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5540	rc = VINF_HM_DOUBLE_FAULT;
5541	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5542	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5543
5544	break;
5545	}
5546
5547	case VMXREFLECTXCPT_TF:
5548	{
5549	rc = VINF_EM_RESET;
5550	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5551	uExitVector));
5552	break;
5553	}
5554
5555	default:
5556	Assert(rc == VINF_SUCCESS);
5557	break;
5558	}
5559	}
5560	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
5561	return rc;
5562	}
5563
5564
5565	/**
5566	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5567	*
5568	* @returns VBox status code.
5569	* @param pVCpu Pointer to the VMCPU.
5570	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5571	* out-of-sync. Make sure to update the required fields
5572	* before using them.
5573	*
5574	* @remarks No-long-jump zone!!!
5575	*/
5576	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5577	{
5578	NOREF(pMixedCtx);
5579
5580	/* Since this can be called from our preemption hook it's safer to make the guest-CR0 update non-preemptible. */
5581	VMMRZCallRing3Disable(pVCpu);
5582	HM_DISABLE_PREEMPT_IF_NEEDED();
5583
5584	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5585	{
5586	uint32_t uVal = 0;
5587	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5588	AssertRCReturn(rc, rc);
5589
5590	uint32_t uShadow = 0;
5591	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5592	AssertRCReturn(rc, rc);
5593
5594	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5595	CPUMSetGuestCR0(pVCpu, uVal);
5596	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5597	}
5598
5599	HM_RESTORE_PREEMPT_IF_NEEDED();
5600	VMMRZCallRing3Enable(pVCpu);
5601
5602	return VINF_SUCCESS;
5603	}
5604
5605
5606	/**
5607	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5608	*
5609	* @returns VBox status code.
5610	* @param pVCpu Pointer to the VMCPU.
5611	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5612	* out-of-sync. Make sure to update the required fields
5613	* before using them.
5614	*
5615	* @remarks No-long-jump zone!!!
5616	*/
5617	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5618	{
5619	NOREF(pMixedCtx);
5620
5621	int rc = VINF_SUCCESS;
5622	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5623	{
5624	uint32_t uVal = 0;
5625	uint32_t uShadow = 0;
5626	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5627	AssertRCReturn(rc, rc);
5628	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5629	AssertRCReturn(rc, rc);
5630
5631	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5632	CPUMSetGuestCR4(pVCpu, uVal);
5633	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5634	}
5635	return rc;
5636	}
5637
5638
5639	/**
5640	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
5641	*
5642	* @returns VBox status code.
5643	* @param pVCpu Pointer to the VMCPU.
5644	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5645	* out-of-sync. Make sure to update the required fields
5646	* before using them.
5647	*
5648	* @remarks No-long-jump zone!!!
5649	*/
5650	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5651	{
5652	int rc = VINF_SUCCESS;
5653	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5654	{
5655	uint64_t u64Val = 0;
5656	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5657	AssertRCReturn(rc, rc);
5658
5659	pMixedCtx->rip = u64Val;
5660	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5661	}
5662	return rc;
5663	}
5664
5665
5666	/**
5667	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
5668	*
5669	* @returns VBox status code.
5670	* @param pVCpu Pointer to the VMCPU.
5671	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5672	* out-of-sync. Make sure to update the required fields
5673	* before using them.
5674	*
5675	* @remarks No-long-jump zone!!!
5676	*/
5677	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5678	{
5679	int rc = VINF_SUCCESS;
5680	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5681	{
5682	uint64_t u64Val = 0;
5683	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5684	AssertRCReturn(rc, rc);
5685
5686	pMixedCtx->rsp = u64Val;
5687	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5688	}
5689	return rc;
5690	}
5691
5692
5693	/**
5694	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5695	*
5696	* @returns VBox status code.
5697	* @param pVCpu Pointer to the VMCPU.
5698	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5699	* out-of-sync. Make sure to update the required fields
5700	* before using them.
5701	*
5702	* @remarks No-long-jump zone!!!
5703	*/
5704	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5705	{
5706	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5707	{
5708	uint32_t uVal = 0;
5709	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5710	AssertRCReturn(rc, rc);
5711
5712	pMixedCtx->eflags.u32 = uVal;
5713	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5714	{
5715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5716	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5717
5718	pMixedCtx->eflags.Bits.u1VM = 0;
5719	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5720	}
5721
5722	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5723	}
5724	return VINF_SUCCESS;
5725	}
5726
5727
5728	/**
5729	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5730	* guest-CPU context.
5731	*/
5732	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5733	{
5734	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5735	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5736	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5737	return rc;
5738	}
5739
5740
5741	/**
5742	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5743	* from the guest-state area in the VMCS.
5744	*
5745	* @param pVCpu Pointer to the VMCPU.
5746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5747	* out-of-sync. Make sure to update the required fields
5748	* before using them.
5749	*
5750	* @remarks No-long-jump zone!!!
5751	*/
5752	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5753	{
5754	uint32_t uIntrState = 0;
5755	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5756	AssertRC(rc);
5757
5758	if (!uIntrState)
5759	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5760	else
5761	{
5762	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5763	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5764	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5765	AssertRC(rc);
5766	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5767	AssertRC(rc);
5768
5769	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5770	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5771	}
5772	}
5773
5774
5775	/**
5776	* Saves the guest's activity state.
5777	*
5778	* @returns VBox status code.
5779	* @param pVCpu Pointer to the VMCPU.
5780	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781	* out-of-sync. Make sure to update the required fields
5782	* before using them.
5783	*
5784	* @remarks No-long-jump zone!!!
5785	*/
5786	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787	{
5788	NOREF(pMixedCtx);
5789	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5790	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
5791	return VINF_SUCCESS;
5792	}
5793
5794
5795	/**
5796	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5797	* the current VMCS into the guest-CPU context.
5798	*
5799	* @returns VBox status code.
5800	* @param pVCpu Pointer to the VMCPU.
5801	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5802	* out-of-sync. Make sure to update the required fields
5803	* before using them.
5804	*
5805	* @remarks No-long-jump zone!!!
5806	*/
5807	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5808	{
5809	int rc = VINF_SUCCESS;
5810	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5811	{
5812	uint32_t u32Val = 0;
5813	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5814	pMixedCtx->SysEnter.cs = u32Val;
5815	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
5816	}
5817
5818	uint64_t u64Val = 0;
5819	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5820	{
5821	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5822	pMixedCtx->SysEnter.eip = u64Val;
5823	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
5824	}
5825	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5826	{
5827	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5828	pMixedCtx->SysEnter.esp = u64Val;
5829	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
5830	}
5831	return rc;
5832	}
5833
5834
5835	/**
5836	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
5837	* the CPU back into the guest-CPU context.
5838	*
5839	* @returns VBox status code.
5840	* @param pVCpu Pointer to the VMCPU.
5841	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5842	* out-of-sync. Make sure to update the required fields
5843	* before using them.
5844	*
5845	* @remarks No-long-jump zone!!!
5846	*/
5847	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5848	{
5849	#if HC_ARCH_BITS == 64
5850	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
5851	{
5852	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
5853	VMMRZCallRing3Disable(pVCpu);
5854	HM_DISABLE_PREEMPT_IF_NEEDED();
5855
5856	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
5857	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
5858	{
5859	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
5860	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5861	}
5862
5863	HM_RESTORE_PREEMPT_IF_NEEDED();
5864	VMMRZCallRing3Enable(pVCpu);
5865	}
5866	else
5867	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5868	#else
5869	NOREF(pMixedCtx);
5870	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
5871	#endif
5872
5873	return VINF_SUCCESS;
5874	}
5875
5876
5877	/**
5878	* Saves the auto load/store'd guest MSRs from the current VMCS into
5879	* the guest-CPU context.
5880	*
5881	* @returns VBox status code.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	*
5887	* @remarks No-long-jump zone!!!
5888	*/
5889	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890	{
5891	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
5892	return VINF_SUCCESS;
5893
5894	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5895	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
5896	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
5897	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
5898	{
5899	switch (pMsr->u32Msr)
5900	{
5901	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5902	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5903	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5904	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5905	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5906	default:
5907	{
5908	AssertFailed();
5909	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5910	}
5911	}
5912	}
5913
5914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
5915	return VINF_SUCCESS;
5916	}
5917
5918
5919	/**
5920	* Saves the guest control registers from the current VMCS into the guest-CPU
5921	* context.
5922	*
5923	* @returns VBox status code.
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	*
5929	* @remarks No-long-jump zone!!!
5930	*/
5931	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5932	{
5933	/* Guest CR0. Guest FPU. */
5934	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5935	AssertRCReturn(rc, rc);
5936
5937	/* Guest CR4. */
5938	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5939	AssertRCReturn(rc, rc);
5940
5941	/* Guest CR2 - updated always during the world-switch or in #PF. */
5942	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5943	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
5944	{
5945	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
5946	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
5947
5948	PVM pVM = pVCpu->CTX_SUFF(pVM);
5949	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5950	\|\| ( pVM->hm.s.fNestedPaging
5951	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5952	{
5953	uint64_t u64Val = 0;
5954	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5955	if (pMixedCtx->cr3 != u64Val)
5956	{
5957	CPUMSetGuestCR3(pVCpu, u64Val);
5958	if (VMMRZCallRing3IsEnabled(pVCpu))
5959	{
5960	PGMUpdateCR3(pVCpu, u64Val);
5961	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5962	}
5963	else
5964	{
5965	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5966	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5967	}
5968	}
5969
5970	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5971	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5972	{
5973	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5974	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5975	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5976	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5977
5978	if (VMMRZCallRing3IsEnabled(pVCpu))
5979	{
5980	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5982	}
5983	else
5984	{
5985	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5986	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5987	}
5988	}
5989	}
5990
5991	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
5992	}
5993
5994	/*
5995	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5996	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5997	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5998	*
5999	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6000	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6001	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6002	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6003	*
6004	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6005	*/
6006	if (VMMRZCallRing3IsEnabled(pVCpu))
6007	{
6008	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6009	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6010
6011	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6012	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6013
6014	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6015	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6016	}
6017
6018	return rc;
6019	}
6020
6021
6022	/**
6023	* Reads a guest segment register from the current VMCS into the guest-CPU
6024	* context.
6025	*
6026	* @returns VBox status code.
6027	* @param pVCpu Pointer to the VMCPU.
6028	* @param idxSel Index of the selector in the VMCS.
6029	* @param idxLimit Index of the segment limit in the VMCS.
6030	* @param idxBase Index of the segment base in the VMCS.
6031	* @param idxAccess Index of the access rights of the segment in the VMCS.
6032	* @param pSelReg Pointer to the segment selector.
6033	*
6034	* @remarks No-long-jump zone!!!
6035	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6036	* macro as that takes care of whether to read from the VMCS cache or
6037	* not.
6038	*/
6039	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6040	PCPUMSELREG pSelReg)
6041	{
6042	NOREF(pVCpu);
6043
6044	uint32_t u32Val = 0;
6045	int rc = VMXReadVmcs32(idxSel, &u32Val);
6046	AssertRCReturn(rc, rc);
6047	pSelReg->Sel = (uint16_t)u32Val;
6048	pSelReg->ValidSel = (uint16_t)u32Val;
6049	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6050
6051	rc = VMXReadVmcs32(idxLimit, &u32Val);
6052	AssertRCReturn(rc, rc);
6053	pSelReg->u32Limit = u32Val;
6054
6055	uint64_t u64Val = 0;
6056	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6057	AssertRCReturn(rc, rc);
6058	pSelReg->u64Base = u64Val;
6059
6060	rc = VMXReadVmcs32(idxAccess, &u32Val);
6061	AssertRCReturn(rc, rc);
6062	pSelReg->Attr.u = u32Val;
6063
6064	/*
6065	* If VT-x marks the segment as unusable, most other bits remain undefined:
6066	* - For CS the L, D and G bits have meaning.
6067	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6068	* - For the remaining data segments no bits are defined.
6069	*
6070	* The present bit and the unusable bit has been observed to be set at the
6071	* same time (the selector was supposed to invalid as we started executing
6072	* a V8086 interrupt in ring-0).
6073	*
6074	* What should be important for the rest of the VBox code, is that the P bit is
6075	* cleared. Some of the other VBox code recognizes the unusable bit, but
6076	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6077	* safe side here, we'll strip off P and other bits we don't care about. If
6078	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6079	*
6080	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6081	*/
6082	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6083	{
6084	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6085
6086	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6087	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6088	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6089
6090	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6091	#ifdef DEBUG_bird
6092	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6093	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6094	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6095	#endif
6096	}
6097	return VINF_SUCCESS;
6098	}
6099
6100
6101	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6102	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6103	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6104	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6105	#else
6106	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6107	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6108	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6109	#endif
6110
6111
6112	/**
6113	* Saves the guest segment registers from the current VMCS into the guest-CPU
6114	* context.
6115	*
6116	* @returns VBox status code.
6117	* @param pVCpu Pointer to the VMCPU.
6118	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6119	* out-of-sync. Make sure to update the required fields
6120	* before using them.
6121	*
6122	* @remarks No-long-jump zone!!!
6123	*/
6124	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6125	{
6126	/* Guest segment registers. */
6127	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6128	{
6129	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6130	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6131	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6132	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6133	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6134	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6135	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6136
6137	/* Restore segment attributes for real-on-v86 mode hack. */
6138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6139	{
6140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6146	}
6147	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6148	}
6149
6150	return VINF_SUCCESS;
6151	}
6152
6153
6154	/**
6155	* Saves the guest descriptor table registers and task register from the current
6156	* VMCS into the guest-CPU context.
6157	*
6158	* @returns VBox status code.
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6161	* out-of-sync. Make sure to update the required fields
6162	* before using them.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6167	{
6168	int rc = VINF_SUCCESS;
6169
6170	/* Guest LDTR. */
6171	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6172	{
6173	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6174	AssertRCReturn(rc, rc);
6175	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6176	}
6177
6178	/* Guest GDTR. */
6179	uint64_t u64Val = 0;
6180	uint32_t u32Val = 0;
6181	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6182	{
6183	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6184	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6185	pMixedCtx->gdtr.pGdt = u64Val;
6186	pMixedCtx->gdtr.cbGdt = u32Val;
6187	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6188	}
6189
6190	/* Guest IDTR. */
6191	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6192	{
6193	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6194	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6195	pMixedCtx->idtr.pIdt = u64Val;
6196	pMixedCtx->idtr.cbIdt = u32Val;
6197	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6198	}
6199
6200	/* Guest TR. */
6201	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6202	{
6203	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6204	AssertRCReturn(rc, rc);
6205
6206	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6207	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6208	{
6209	rc = VMXLOCAL_READ_SEG(TR, tr);
6210	AssertRCReturn(rc, rc);
6211	}
6212	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6213	}
6214	return rc;
6215	}
6216
6217	#undef VMXLOCAL_READ_SEG
6218
6219
6220	/**
6221	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6222	* context.
6223	*
6224	* @returns VBox status code.
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	*
6230	* @remarks No-long-jump zone!!!
6231	*/
6232	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6233	{
6234	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6235	{
6236	if (!pVCpu->hm.s.fUsingHyperDR7)
6237	{
6238	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6239	uint32_t u32Val;
6240	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6241	pMixedCtx->dr[7] = u32Val;
6242	}
6243
6244	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6245	}
6246	return VINF_SUCCESS;
6247	}
6248
6249
6250	/**
6251	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6252	*
6253	* @returns VBox status code.
6254	* @param pVCpu Pointer to the VMCPU.
6255	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256	* out-of-sync. Make sure to update the required fields
6257	* before using them.
6258	*
6259	* @remarks No-long-jump zone!!!
6260	*/
6261	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262	{
6263	NOREF(pMixedCtx);
6264
6265	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6266	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6267	return VINF_SUCCESS;
6268	}
6269
6270
6271	/**
6272	* Saves the entire guest state from the currently active VMCS into the
6273	* guest-CPU context. This essentially VMREADs all guest-data.
6274	*
6275	* @returns VBox status code.
6276	* @param pVCpu Pointer to the VMCPU.
6277	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6278	* out-of-sync. Make sure to update the required fields
6279	* before using them.
6280	*/
6281	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282	{
6283	Assert(pVCpu);
6284	Assert(pMixedCtx);
6285
6286	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6287	return VINF_SUCCESS;
6288
6289	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6290	again on the ring-3 callback path, there is no real need to. */
6291	if (VMMRZCallRing3IsEnabled(pVCpu))
6292	VMMR0LogFlushDisable(pVCpu);
6293	else
6294	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6295	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6296
6297	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6299
6300	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6302
6303	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6305
6306	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6323
6324	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6326
6327	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6328	("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6329
6330	if (VMMRZCallRing3IsEnabled(pVCpu))
6331	VMMR0LogFlushEnable(pVCpu);
6332
6333	return rc;
6334	}
6335
6336
6337	/**
6338	* Check per-VM and per-VCPU force flag actions that require us to go back to
6339	* ring-3 for one reason or another.
6340	*
6341	* @returns VBox status code (information status code included).
6342	* @retval VINF_SUCCESS if we don't have any actions that require going back to
6343	* ring-3.
6344	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6345	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6346	* interrupts)
6347	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6348	* all EMTs to be in ring-3.
6349	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6350	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6351	* to the EM loop.
6352	*
6353	* @param pVM Pointer to the VM.
6354	* @param pVCpu Pointer to the VMCPU.
6355	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6356	* out-of-sync. Make sure to update the required fields
6357	* before using them.
6358	*/
6359	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6360	{
6361	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6362
6363	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6364	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6365	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6366	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6367	{
6368	/* We need the control registers now, make sure the guest-CPU context is updated. */
6369	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6370	AssertRCReturn(rc3, rc3);
6371
6372	/* Pending HM CR3 sync. */
6373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6374	{
6375	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6376	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
6377	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6378	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6379	}
6380
6381	/* Pending HM PAE PDPEs. */
6382	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6383	{
6384	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6385	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6386	}
6387
6388	/* Pending PGM C3 sync. */
6389	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6390	{
6391	int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6392	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6393	if (rc2 != VINF_SUCCESS)
6394	{
6395	AssertRC(rc2);
6396	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6397	return rc2;
6398	}
6399	}
6400
6401	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6402	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6403	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6404	{
6405	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6406	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6407	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6408	return rc2;
6409	}
6410
6411	/* Pending VM request packets, such as hardware interrupts. */
6412	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6413	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6414	{
6415	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6416	return VINF_EM_PENDING_REQUEST;
6417	}
6418
6419	/* Pending PGM pool flushes. */
6420	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6421	{
6422	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6423	return VINF_PGM_POOL_FLUSH_PENDING;
6424	}
6425
6426	/* Pending DMA requests. */
6427	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6428	{
6429	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6430	return VINF_EM_RAW_TO_R3;
6431	}
6432	}
6433
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Converts any TRPM trap into a pending HM event. This is typically used when
6440	* entering from ring-3 (not longjmp returns).
6441	*
6442	* @param pVCpu Pointer to the VMCPU.
6443	*/
6444	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6445	{
6446	Assert(TRPMHasTrap(pVCpu));
6447	Assert(!pVCpu->hm.s.Event.fPending);
6448
6449	uint8_t uVector;
6450	TRPMEVENT enmTrpmEvent;
6451	RTGCUINT uErrCode;
6452	RTGCUINTPTR GCPtrFaultAddress;
6453	uint8_t cbInstr;
6454
6455	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6456	AssertRC(rc);
6457
6458	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6459	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6460	if (enmTrpmEvent == TRPM_TRAP)
6461	{
6462	switch (uVector)
6463	{
6464	case X86_XCPT_BP:
6465	case X86_XCPT_OF:
6466	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6467	break;
6468
6469	case X86_XCPT_PF:
6470	case X86_XCPT_DF:
6471	case X86_XCPT_TS:
6472	case X86_XCPT_NP:
6473	case X86_XCPT_SS:
6474	case X86_XCPT_GP:
6475	case X86_XCPT_AC:
6476	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6477	/* no break! */
6478	default:
6479	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6480	break;
6481	}
6482	}
6483	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6484	{
6485	if (uVector == X86_XCPT_NMI)
6486	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6487	else
6488	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6489	}
6490	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6491	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6492	else
6493	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6494
6495	rc = TRPMResetTrap(pVCpu);
6496	AssertRC(rc);
6497	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6498	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6499
6500	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6501	STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6502	}
6503
6504
6505	/**
6506	* Converts any pending HM event into a TRPM trap. Typically used when leaving
6507	* VT-x to execute any instruction.
6508	*
6509	* @param pvCpu Pointer to the VMCPU.
6510	*/
6511	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6512	{
6513	Assert(pVCpu->hm.s.Event.fPending);
6514
6515	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6516	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6517	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6518	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6519
6520	/* If a trap was already pending, we did something wrong! */
6521	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6522
6523	TRPMEVENT enmTrapType;
6524	switch (uVectorType)
6525	{
6526	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6527	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6528	enmTrapType = TRPM_HARDWARE_INT;
6529	break;
6530
6531	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6532	enmTrapType = TRPM_SOFTWARE_INT;
6533	break;
6534
6535	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6536	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6537	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6538	enmTrapType = TRPM_TRAP;
6539	break;
6540
6541	default:
6542	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6543	enmTrapType = TRPM_32BIT_HACK;
6544	break;
6545	}
6546
6547	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6548
6549	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6550	AssertRC(rc);
6551
6552	if (fErrorCodeValid)
6553	TRPMSetErrorCode(pVCpu, uErrorCode);
6554
6555	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6556	&& uVector == X86_XCPT_PF)
6557	{
6558	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6559	}
6560	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6561	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6562	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6563	{
6564	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6565	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
6566	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6567	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6568	}
6569	pVCpu->hm.s.Event.fPending = false;
6570	}
6571
6572
6573	/**
6574	* Does the necessary state syncing before returning to ring-3 for any reason
6575	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6576	*
6577	* @returns VBox status code.
6578	* @param pVM Pointer to the VM.
6579	* @param pVCpu Pointer to the VMCPU.
6580	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6581	* be out-of-sync. Make sure to update the required
6582	* fields before using them.
6583	* @param fSaveGuestState Whether to save the guest state or not.
6584	*
6585	* @remarks If you modify code here, make sure to check whether
6586	* hmR0VmxCallRing3Callback() needs to be updated too!!!
6587	* @remarks No-long-jmp zone!!!
6588	*/
6589	static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6590	{
6591	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6592	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6593
6594	RTCPUID idCpu = RTMpCpuId();
6595	Log4Func(("HostCpuId=%u\n", idCpu));
6596
6597	/* Save the guest state if necessary. */
6598	if ( fSaveGuestState
6599	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6600	{
6601	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6602	AssertRCReturn(rc, rc);
6603	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6604	}
6605
6606	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
6607	if (CPUMIsGuestFPUStateActive(pVCpu))
6608	{
6609	/* We shouldn't reload CR0 without saving it first. */
6610	if (!fSaveGuestState)
6611	{
6612	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6613	AssertRCReturn(rc, rc);
6614	}
6615	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6616	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6617	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6618	}
6619
6620	/* Restore host debug registers if necessary and resync on next R0 reentry. */
6621	#ifdef VBOX_STRICT
6622	if (CPUMIsHyperDebugStateActive(pVCpu))
6623	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6624	#endif
6625	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6626	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6627	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6628	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6629
6630	#if HC_ARCH_BITS == 64
6631	/* Restore host-state bits that VT-x only restores partially. */
6632	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6633	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6634	{
6635	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6636	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6637	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6638	}
6639	#endif
6640
6641	#if HC_ARCH_BITS == 64
6642	/* Restore the host MSRs as we're leaving VT-x context. */
6643	if ( pVM->hm.s.fAllow64BitGuests
6644	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6645	{
6646	/* We shouldn't reload the guest MSRs without saving it first. */
6647	if (!fSaveGuestState)
6648	{
6649	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6650	AssertRCReturn(rc, rc);
6651	}
6652	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6653	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6654	Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6655	}
6656	#endif
6657
6658	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6659	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6660	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6661	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6662	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6663	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6664	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6666
6667	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6668
6669	/** @todo This partially defeats the purpose of having preemption hooks.
6670	* The problem is, deregistering the hooks should be moved to a place that
6671	* lasts until the EMT is about to be destroyed not everytime while leaving HM
6672	* context.
6673	*/
6674	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6675	{
6676	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6677	AssertRCReturn(rc, rc);
6678
6679	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6680	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6681	}
6682	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6683	NOREF(idCpu);
6684
6685	return VINF_SUCCESS;
6686	}
6687
6688
6689	/**
6690	* Leaves the VT-x session.
6691	*
6692	* @returns VBox status code.
6693	* @param pVM Pointer to the VM.
6694	* @param pVCpu Pointer to the VMCPU.
6695	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6696	* out-of-sync. Make sure to update the required fields
6697	* before using them.
6698	*
6699	* @remarks No-long-jmp zone!!!
6700	*/
6701	DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6702	{
6703	HM_DISABLE_PREEMPT_IF_NEEDED();
6704	HMVMX_ASSERT_CPU_SAFE();
6705	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6706	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6707
6708	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6709	and done this from the VMXR0ThreadCtxCallback(). */
6710	if (!pVCpu->hm.s.fLeaveDone)
6711	{
6712	int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6713	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6714	pVCpu->hm.s.fLeaveDone = true;
6715	}
6716
6717	/* Deregister hook now that we've left HM context before re-enabling preemption. */
6718	/** @todo This is bad. Deregistering here means we need to VMCLEAR always
6719	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
6720	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6721	VMMR0ThreadCtxHooksDeregister(pVCpu);
6722
6723	/* Leave HM context. This takes care of local init (term). */
6724	int rc = HMR0LeaveCpu(pVCpu);
6725
6726	HM_RESTORE_PREEMPT_IF_NEEDED();
6727
6728	return rc;
6729	}
6730
6731
6732	/**
6733	* Does the necessary state syncing before doing a longjmp to ring-3.
6734	*
6735	* @returns VBox status code.
6736	* @param pVM Pointer to the VM.
6737	* @param pVCpu Pointer to the VMCPU.
6738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6739	* out-of-sync. Make sure to update the required fields
6740	* before using them.
6741	*
6742	* @remarks No-long-jmp zone!!!
6743	*/
6744	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6745	{
6746	return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6747	}
6748
6749
6750	/**
6751	* Take necessary actions before going back to ring-3.
6752	*
6753	* An action requires us to go back to ring-3. This function does the necessary
6754	* steps before we can safely return to ring-3. This is not the same as longjmps
6755	* to ring-3, this is voluntary and prepares the guest so it may continue
6756	* executing outside HM (recompiler/IEM).
6757	*
6758	* @returns VBox status code.
6759	* @param pVM Pointer to the VM.
6760	* @param pVCpu Pointer to the VMCPU.
6761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6762	* out-of-sync. Make sure to update the required fields
6763	* before using them.
6764	* @param rcExit The reason for exiting to ring-3. Can be
6765	* VINF_VMM_UNKNOWN_RING3_CALL.
6766	*/
6767	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6768	{
6769	Assert(pVM);
6770	Assert(pVCpu);
6771	Assert(pMixedCtx);
6772	HMVMX_ASSERT_PREEMPT_SAFE();
6773
6774	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6775	{
6776	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6777	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
6778	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6779	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6780	}
6781
6782	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6783	VMMRZCallRing3Disable(pVCpu);
6784	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6785
6786	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6787	if (pVCpu->hm.s.Event.fPending)
6788	{
6789	hmR0VmxPendingEventToTrpmTrap(pVCpu);
6790	Assert(!pVCpu->hm.s.Event.fPending);
6791	}
6792
6793	/* Save guest state and restore host state bits. */
6794	int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6795	AssertRCReturn(rc, rc);
6796	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6797
6798	/* Sync recompiler state. */
6799	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6800	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6801	\| CPUM_CHANGED_LDTR
6802	\| CPUM_CHANGED_GDTR
6803	\| CPUM_CHANGED_IDTR
6804	\| CPUM_CHANGED_TR
6805	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
6806	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6807	if ( pVM->hm.s.fNestedPaging
6808	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
6809	{
6810	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6811	}
6812
6813	Assert(!pVCpu->hm.s.fClearTrapFlag);
6814
6815	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6816	if (rcExit != VINF_EM_RAW_INTERRUPT)
6817	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6818
6819	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6820
6821	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6822	VMMRZCallRing3RemoveNotification(pVCpu);
6823	VMMRZCallRing3Enable(pVCpu);
6824
6825	return rc;
6826	}
6827
6828
6829	/**
6830	* VMMRZCallRing3() callback wrapper which saves the guest state before we
6831	* longjump to ring-3 and possibly get preempted.
6832	*
6833	* @returns VBox status code.
6834	* @param pVCpu Pointer to the VMCPU.
6835	* @param enmOperation The operation causing the ring-3 longjump.
6836	* @param pvUser Opaque pointer to the guest-CPU context. The data
6837	* may be out-of-sync. Make sure to update the required
6838	* fields before using them.
6839	*
6840	* @remarks If you modify code here, make sure to check whether
6841	* hmR0VmxLeave() needs to be updated too!!!
6842	*/
6843	DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6844	{
6845	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6846	{
6847	/* If anything here asserts or fails, good luck. */
6848	VMMRZCallRing3RemoveNotification(pVCpu);
6849	VMMRZCallRing3Disable(pVCpu);
6850	HM_DISABLE_PREEMPT_IF_NEEDED();
6851
6852	PVM pVM = pVCpu->CTX_SUFF(pVM);
6853	if (CPUMIsGuestFPUStateActive(pVCpu))
6854	CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
6855
6856	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6857
6858	#if HC_ARCH_BITS == 64
6859	/* Restore host-state bits that VT-x only restores partially. */
6860	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6861	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6862	{
6863	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6864	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6865	}
6866	#endif
6867
6868	#if HC_ARCH_BITS == 64
6869	/* Restore the host MSRs as we're leaving VT-x context. */
6870	if ( pVM->hm.s.fAllow64BitGuests
6871	&& pVCpu->hm.s.vmx.fRestoreHostMsrs)
6872	{
6873	hmR0VmxLazyRestoreHostMsrs(pVCpu);
6874	}
6875	#endif
6876	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6877	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6878	{
6879	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6880	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6881	}
6882
6883	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6884	VMMR0ThreadCtxHooksDeregister(pVCpu);
6885
6886	HMR0LeaveCpu(pVCpu);
6887	HM_RESTORE_PREEMPT_IF_NEEDED();
6888	return VINF_SUCCESS;
6889	}
6890
6891	Assert(pVCpu);
6892	Assert(pvUser);
6893	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6894	HMVMX_ASSERT_PREEMPT_SAFE();
6895
6896	VMMRZCallRing3Disable(pVCpu);
6897	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6898
6899	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6900	enmOperation));
6901
6902	int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6903	AssertRCReturn(rc, rc);
6904
6905	VMMRZCallRing3Enable(pVCpu);
6906	return VINF_SUCCESS;
6907	}
6908
6909
6910	/**
6911	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6912	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6913	*
6914	* @param pVCpu Pointer to the VMCPU.
6915	*/
6916	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6917	{
6918	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6919	{
6920	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6921	{
6922	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6923	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6924	AssertRC(rc);
6925	}
6926	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6927	}
6928
6929
6930	/**
6931	* Evaluates the event to be delivered to the guest and sets it as the pending
6932	* event.
6933	*
6934	* @param pVCpu Pointer to the VMCPU.
6935	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6936	* out-of-sync. Make sure to update the required fields
6937	* before using them.
6938	*/
6939	static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6940	{
6941	Assert(!pVCpu->hm.s.Event.fPending);
6942
6943	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6944	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6945	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6946	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6947
6948	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
6949	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6950	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6951	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6952	Assert(!TRPMHasTrap(pVCpu));
6953
6954	/** @todo SMI. SMIs take priority over NMIs. */
6955	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6956	{
6957	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6958	if ( !fBlockMovSS
6959	&& !fBlockSti)
6960	{
6961	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6962	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6963	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6964	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6965
6966	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddres */);
6967	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6968	}
6969	else
6970	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6971	}
6972	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
6973	&& !pVCpu->hm.s.fSingleInstruction)
6974	{
6975	/*
6976	* Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6977	* the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6978	* evaluated here and not set as pending, solely based on the force-flags.
6979	*/
6980	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6981	AssertRC(rc);
6982	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6983	if ( !fBlockInt
6984	&& !fBlockSti
6985	&& !fBlockMovSS)
6986	{
6987	uint8_t u8Interrupt;
6988	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6989	if (RT_SUCCESS(rc))
6990	{
6991	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6992	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6993	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6994
6995	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
6996	}
6997	else
6998	{
6999	/** @todo Does this actually happen? If not turn it into an assertion. */
7000	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
7001	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7002	}
7003	}
7004	else
7005	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7006	}
7007	}
7008
7009
7010	/**
7011	* Injects any pending events into the guest if the guest is in a state to
7012	* receive them.
7013	*
7014	* @returns VBox status code (informational status codes included).
7015	* @param pVCpu Pointer to the VMCPU.
7016	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7017	* out-of-sync. Make sure to update the required fields
7018	* before using them.
7019	*/
7020	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7021	{
7022	HMVMX_ASSERT_PREEMPT_SAFE();
7023	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7024
7025	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7026	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7027	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7028	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7029
7030	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7031	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7032	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7033	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7034	Assert(!TRPMHasTrap(pVCpu));
7035
7036	int rc = VINF_SUCCESS;
7037	if (pVCpu->hm.s.Event.fPending)
7038	{
7039	#if defined(VBOX_STRICT) \|\| defined(VBOX_WITH_STATISTICS)
7040	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7041	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7042	{
7043	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7044	AssertRCReturn(rc, rc);
7045	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7046	Assert(!fBlockInt);
7047	Assert(!fBlockSti);
7048	Assert(!fBlockMovSS);
7049	}
7050	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7051	{
7052	Assert(!fBlockSti);
7053	Assert(!fBlockMovSS);
7054	}
7055	#endif
7056	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo));
7057	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7058	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7059	AssertRCReturn(rc, rc);
7060
7061	/* Update the interruptibility-state as it could have been changed by
7062	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7063	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7064	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7065
7066	#ifdef VBOX_WITH_STATISTICS
7067	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7068	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7069	else
7070	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7071	#endif
7072	}
7073
7074	/* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7075	if ( !pVCpu->hm.s.fSingleInstruction
7076	&& !DBGFIsStepping(pVCpu))
7077	{
7078	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7079	AssertRCReturn(rc2, rc2);
7080	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7081	{
7082	/*
7083	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7084	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7085	* See Intel spec. 27.3.4 "Saving Non-Register State".
7086	*/
7087	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7088	AssertRCReturn(rc2, rc2);
7089	}
7090	}
7091	else
7092	{
7093	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
7094	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
7095	uIntrState = 0;
7096	}
7097
7098	/*
7099	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7100	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7101	*/
7102	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7103	AssertRC(rc2);
7104
7105	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7106	NOREF(fBlockMovSS); NOREF(fBlockSti);
7107	return rc;
7108	}
7109
7110
7111	/**
7112	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7113	*
7114	* @param pVCpu Pointer to the VMCPU.
7115	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7116	* out-of-sync. Make sure to update the required fields
7117	* before using them.
7118	*/
7119	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7120	{
7121	NOREF(pMixedCtx);
7122	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7123	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7124	}
7125
7126
7127	/**
7128	* Injects a double-fault (#DF) exception into the VM.
7129	*
7130	* @returns VBox status code (informational status code included).
7131	* @param pVCpu Pointer to the VMCPU.
7132	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7133	* out-of-sync. Make sure to update the required fields
7134	* before using them.
7135	*/
7136	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7137	{
7138	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7139	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7140	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7141	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
7142	puIntrState);
7143	}
7144
7145
7146	/**
7147	* Sets a debug (#DB) exception as pending-for-injection into the VM.
7148	*
7149	* @param pVCpu Pointer to the VMCPU.
7150	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7151	* out-of-sync. Make sure to update the required fields
7152	* before using them.
7153	*/
7154	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7155	{
7156	NOREF(pMixedCtx);
7157	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7158	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7159	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7160	}
7161
7162
7163	/**
7164	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
7165	*
7166	* @param pVCpu Pointer to the VMCPU.
7167	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7168	* out-of-sync. Make sure to update the required fields
7169	* before using them.
7170	* @param cbInstr The value of RIP that is to be pushed on the guest
7171	* stack.
7172	*/
7173	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7174	{
7175	NOREF(pMixedCtx);
7176	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7177	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7178	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7179	}
7180
7181
7182	/**
7183	* Injects a general-protection (#GP) fault into the VM.
7184	*
7185	* @returns VBox status code (informational status code included).
7186	* @param pVCpu Pointer to the VMCPU.
7187	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7188	* out-of-sync. Make sure to update the required fields
7189	* before using them.
7190	* @param u32ErrorCode The error code associated with the #GP.
7191	*/
7192	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7193	uint32_t *puIntrState)
7194	{
7195	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7196	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7197	if (fErrorCodeValid)
7198	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7199	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
7200	puIntrState);
7201	}
7202
7203
7204	/**
7205	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
7206	*
7207	* @param pVCpu Pointer to the VMCPU.
7208	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7209	* out-of-sync. Make sure to update the required fields
7210	* before using them.
7211	* @param uVector The software interrupt vector number.
7212	* @param cbInstr The value of RIP that is to be pushed on the guest
7213	* stack.
7214	*/
7215	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7216	{
7217	NOREF(pMixedCtx);
7218	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7219	if ( uVector == X86_XCPT_BP
7220	\|\| uVector == X86_XCPT_OF)
7221	{
7222	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7223	}
7224	else
7225	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7226	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
7227	}
7228
7229
7230	/**
7231	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7232	* stack.
7233	*
7234	* @returns VBox status code (information status code included).
7235	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7236	* @param pVM Pointer to the VM.
7237	* @param pMixedCtx Pointer to the guest-CPU context.
7238	* @param uValue The value to push to the guest stack.
7239	*/
7240	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7241	{
7242	/*
7243	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7244	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7245	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7246	*/
7247	if (pMixedCtx->sp == 1)
7248	return VINF_EM_RESET;
7249	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7250	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7251	AssertRCReturn(rc, rc);
7252	return rc;
7253	}
7254
7255
7256	/**
7257	* Injects an event into the guest upon VM-entry by updating the relevant fields
7258	* in the VM-entry area in the VMCS.
7259	*
7260	* @returns VBox status code (informational error codes included).
7261	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7262	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7263	*
7264	* @param pVCpu Pointer to the VMCPU.
7265	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7266	* be out-of-sync. Make sure to update the required
7267	* fields before using them.
7268	* @param u64IntInfo The VM-entry interruption-information field.
7269	* @param cbInstr The VM-entry instruction length in bytes (for
7270	* software interrupts, exceptions and privileged
7271	* software exceptions).
7272	* @param u32ErrCode The VM-entry exception error code.
7273	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7274	* @param puIntrState Pointer to the current guest interruptibility-state.
7275	* This interruptibility-state will be updated if
7276	* necessary. This cannot not be NULL.
7277	*
7278	* @remarks Requires CR0!
7279	* @remarks No-long-jump zone!!!
7280	*/
7281	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7282	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7283	{
7284	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7285	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7286	Assert(puIntrState);
7287	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7288
7289	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7290	const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7291
7292	#ifdef VBOX_STRICT
7293	/* Validate the error-code-valid bit for hardware exceptions. */
7294	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7295	{
7296	switch (uVector)
7297	{
7298	case X86_XCPT_PF:
7299	case X86_XCPT_DF:
7300	case X86_XCPT_TS:
7301	case X86_XCPT_NP:
7302	case X86_XCPT_SS:
7303	case X86_XCPT_GP:
7304	case X86_XCPT_AC:
7305	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7306	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7307	/* fallthru */
7308	default:
7309	break;
7310	}
7311	}
7312	#endif
7313
7314	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
7315	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7316	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7317
7318	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7319
7320	/* We require CR0 to check if the guest is in real-mode. */
7321	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7322	AssertRCReturn(rc, rc);
7323
7324	/*
7325	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7326	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7327	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7328	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7329	*/
7330	if (CPUMIsGuestInRealModeEx(pMixedCtx))
7331	{
7332	PVM pVM = pVCpu->CTX_SUFF(pVM);
7333	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7334	{
7335	Assert(PDMVmmDevHeapIsEnabled(pVM));
7336	Assert(pVM->hm.s.vmx.pRealModeTSS);
7337
7338	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7339	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7340	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7341	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7342	AssertRCReturn(rc, rc);
7343	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7344
7345	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7346	const size_t cbIdtEntry = sizeof(X86IDTR16);
7347	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7348	{
7349	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7350	if (uVector == X86_XCPT_DF)
7351	return VINF_EM_RESET;
7352	else if (uVector == X86_XCPT_GP)
7353	{
7354	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7355	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7356	}
7357
7358	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7359	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7360	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
7361	}
7362
7363	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7364	uint16_t uGuestIp = pMixedCtx->ip;
7365	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7366	{
7367	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7368	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
7369	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7370	}
7371	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7372	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7373
7374	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7375	X86IDTR16 IdtEntry;
7376	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7377	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7378	AssertRCReturn(rc, rc);
7379
7380	/* Construct the stack frame for the interrupt/exception handler. */
7381	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7382	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7383	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7384	AssertRCReturn(rc, rc);
7385
7386	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
7387	if (rc == VINF_SUCCESS)
7388	{
7389	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
7390	pMixedCtx->rip = IdtEntry.offSel;
7391	pMixedCtx->cs.Sel = IdtEntry.uSel;
7392	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7393	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7394	&& uVector == X86_XCPT_PF)
7395	{
7396	pMixedCtx->cr2 = GCPtrFaultAddress;
7397	}
7398
7399	/* If any other guest-state bits are changed here, make sure to update
7400	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7401	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7402	\| HM_CHANGED_GUEST_RIP
7403	\| HM_CHANGED_GUEST_RFLAGS
7404	\| HM_CHANGED_GUEST_RSP);
7405
7406	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7407	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7408	{
7409	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7410	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7411	Log4(("Clearing inhibition due to STI.\n"));
7412	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7413	}
7414	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7415
7416	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7417	it, if we are returning to ring-3 before executing guest code. */
7418	pVCpu->hm.s.Event.fPending = false;
7419	}
7420	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
7421	return rc;
7422	}
7423	else
7424	{
7425	/*
7426	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7427	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7428	*/
7429	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7430	}
7431	}
7432
7433	/* Validate. */
7434	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7435	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7436	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7437
7438	/* Inject. */
7439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7440	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7441	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7442	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7443
7444	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7445	&& uVector == X86_XCPT_PF)
7446	{
7447	pMixedCtx->cr2 = GCPtrFaultAddress;
7448	}
7449
7450	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7451	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7452
7453	AssertRCReturn(rc, rc);
7454	return rc;
7455	}
7456
7457
7458	/**
7459	* Clears the interrupt-window exiting control in the VMCS and if necessary
7460	* clears the current event in the VMCS as well.
7461	*
7462	* @returns VBox status code.
7463	* @param pVCpu Pointer to the VMCPU.
7464	*
7465	* @remarks Use this function only to clear events that have not yet been
7466	* delivered to the guest but are injected in the VMCS!
7467	* @remarks No-long-jump zone!!!
7468	*/
7469	static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7470	{
7471	int rc;
7472	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7473
7474	/* Clear interrupt-window exiting control. */
7475	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7476	{
7477	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7478	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7479	AssertRC(rc);
7480	}
7481
7482	if (!pVCpu->hm.s.Event.fPending)
7483	return;
7484
7485	#ifdef VBOX_STRICT
7486	uint32_t u32EntryInfo;
7487	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7488	AssertRC(rc);
7489	Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7490	#endif
7491
7492	/* Clear the entry-interruption field (including the valid bit). */
7493	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7494	AssertRC(rc);
7495
7496	/* Clear the pending debug exception field. */
7497	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7498	AssertRC(rc);
7499	}
7500
7501
7502	/**
7503	* Enters the VT-x session.
7504	*
7505	* @returns VBox status code.
7506	* @param pVM Pointer to the VM.
7507	* @param pVCpu Pointer to the VMCPU.
7508	* @param pCpu Pointer to the CPU info struct.
7509	*/
7510	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7511	{
7512	AssertPtr(pVM);
7513	AssertPtr(pVCpu);
7514	Assert(pVM->hm.s.vmx.fSupported);
7515	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7516	NOREF(pCpu); NOREF(pVM);
7517
7518	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7519	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7520
7521	#ifdef VBOX_STRICT
7522	/* Make sure we're in VMX root mode. */
7523	RTCCUINTREG u32HostCR4 = ASMGetCR4();
7524	if (!(u32HostCR4 & X86_CR4_VMXE))
7525	{
7526	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7527	return VERR_VMX_X86_CR4_VMXE_CLEARED;
7528	}
7529	#endif
7530
7531	/*
7532	* Load the VCPU's VMCS as the current (and active) one.
7533	*/
7534	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7535	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7536	if (RT_FAILURE(rc))
7537	return rc;
7538
7539	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7540	pVCpu->hm.s.fLeaveDone = false;
7541	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7542
7543	return VINF_SUCCESS;
7544	}
7545
7546
7547	/**
7548	* The thread-context callback (only on platforms which support it).
7549	*
7550	* @param enmEvent The thread-context event.
7551	* @param pVCpu Pointer to the VMCPU.
7552	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7553	* @thread EMT(pVCpu)
7554	*/
7555	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7556	{
7557	NOREF(fGlobalInit);
7558
7559	switch (enmEvent)
7560	{
7561	case RTTHREADCTXEVENT_PREEMPTING:
7562	{
7563	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7564	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7565	VMCPU_ASSERT_EMT(pVCpu);
7566
7567	PVM pVM = pVCpu->CTX_SUFF(pVM);
7568	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7569
7570	/* No longjmps (logger flushes, locks) in this fragile context. */
7571	VMMRZCallRing3Disable(pVCpu);
7572	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7573
7574	/*
7575	* Restore host-state (FPU, debug etc.)
7576	*/
7577	if (!pVCpu->hm.s.fLeaveDone)
7578	{
7579	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7580	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7581	hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7582	pVCpu->hm.s.fLeaveDone = true;
7583	}
7584
7585	/* Leave HM context, takes care of local init (term). */
7586	int rc = HMR0LeaveCpu(pVCpu);
7587	AssertRC(rc); NOREF(rc);
7588
7589	/* Restore longjmp state. */
7590	VMMRZCallRing3Enable(pVCpu);
7591	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7592	break;
7593	}
7594
7595	case RTTHREADCTXEVENT_RESUMED:
7596	{
7597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7598	Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7599	VMCPU_ASSERT_EMT(pVCpu);
7600
7601	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7602	VMMRZCallRing3Disable(pVCpu);
7603	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7604
7605	/* Initialize the bare minimum state required for HM. This takes care of
7606	initializing VT-x if necessary (onlined CPUs, local init etc.) */
7607	int rc = HMR0EnterCpu(pVCpu);
7608	AssertRC(rc);
7609	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
7610
7611	/* Load the active VMCS as the current one. */
7612	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7613	{
7614	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7615	AssertRC(rc); NOREF(rc);
7616	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7617	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7618	}
7619	pVCpu->hm.s.fLeaveDone = false;
7620
7621	/* Restore longjmp state. */
7622	VMMRZCallRing3Enable(pVCpu);
7623	break;
7624	}
7625
7626	default:
7627	break;
7628	}
7629	}
7630
7631
7632	/**
7633	* Saves the host state in the VMCS host-state.
7634	* Sets up the VM-exit MSR-load area.
7635	*
7636	* The CPU state will be loaded from these fields on every successful VM-exit.
7637	*
7638	* @returns VBox status code.
7639	* @param pVM Pointer to the VM.
7640	* @param pVCpu Pointer to the VMCPU.
7641	*
7642	* @remarks No-long-jump zone!!!
7643	*/
7644	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7645	{
7646	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7647
7648	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7649	return VINF_SUCCESS;
7650
7651	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7652	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7653
7654	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7655	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7656
7657	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7658	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7659
7660	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7661	return rc;
7662	}
7663
7664
7665	/**
7666	* Saves the host state in the VMCS host-state.
7667	*
7668	* @returns VBox status code.
7669	* @param pVM Pointer to the VM.
7670	* @param pVCpu Pointer to the VMCPU.
7671	*
7672	* @remarks No-long-jump zone!!!
7673	*/
7674	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7675	{
7676	AssertPtr(pVM);
7677	AssertPtr(pVCpu);
7678
7679	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7680
7681	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7682	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7683	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7684	return hmR0VmxSaveHostState(pVM, pVCpu);
7685	}
7686
7687
7688	/**
7689	* Loads the guest state into the VMCS guest-state area. The CPU state will be
7690	* loaded from these fields on every successful VM-entry.
7691	*
7692	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7693	* Sets up the VM-entry controls.
7694	* Sets up the appropriate VMX non-root function to execute guest code based on
7695	* the guest CPU mode.
7696	*
7697	* @returns VBox status code.
7698	* @param pVM Pointer to the VM.
7699	* @param pVCpu Pointer to the VMCPU.
7700	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7701	* out-of-sync. Make sure to update the required fields
7702	* before using them.
7703	*
7704	* @remarks No-long-jump zone!!!
7705	*/
7706	static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7707	{
7708	AssertPtr(pVM);
7709	AssertPtr(pVCpu);
7710	AssertPtr(pMixedCtx);
7711	HMVMX_ASSERT_PREEMPT_SAFE();
7712
7713	#ifdef LOG_ENABLED
7714	/** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7715	* probably not initialized yet? Anyway this will do for now.
7716	*
7717	* Update: Should be possible once VMXR0LoadGuestState() is removed as an
7718	* interface and disable ring-3 calls when thread-context hooks are not
7719	* available. */
7720	bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7721	VMMR0LogFlushDisable(pVCpu);
7722	#endif
7723
7724	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7725
7726	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7727
7728	/* Determine real-on-v86 mode. */
7729	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7730	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7731	&& CPUMIsGuestInRealModeEx(pMixedCtx))
7732	{
7733	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7734	}
7735
7736	/*
7737	* Load the guest-state into the VMCS.
7738	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7739	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7740	*/
7741	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7742	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7743
7744	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7745	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7746	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7747
7748	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7749	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7750	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7751
7752	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7753	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7754
7755	rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7756	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7757
7758	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7759	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7760	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7761
7762	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7763	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7764
7765	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7766	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7767
7768	/*
7769	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7770	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7771	*/
7772	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7773	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7774
7775	/* Clear any unused and reserved bits. */
7776	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7777
7778	#ifdef LOG_ENABLED
7779	/* Only reenable log-flushing if the caller has it enabled. */
7780	if (!fCallerDisabledLogFlush)
7781	VMMR0LogFlushEnable(pVCpu);
7782	#endif
7783
7784	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7785	return rc;
7786	}
7787
7788
7789	/**
7790	* Loads the state shared between the host and guest into the VMCS.
7791	*
7792	* @param pVM Pointer to the VM.
7793	* @param pVCpu Pointer to the VMCPU.
7794	* @param pCtx Pointer to the guest-CPU context.
7795	*
7796	* @remarks No-long-jump zone!!!
7797	*/
7798	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7799	{
7800	NOREF(pVM);
7801
7802	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7803	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7804
7805	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7806	{
7807	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7808	AssertRC(rc);
7809	}
7810
7811	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7812	{
7813	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7814	AssertRC(rc);
7815
7816	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7817	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7818	{
7819	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7820	AssertRC(rc);
7821	}
7822	}
7823
7824	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
7825	{
7826	#if HC_ARCH_BITS == 64
7827	if (pVM->hm.s.fAllow64BitGuests)
7828	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
7829	#endif
7830	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
7831	}
7832
7833	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7834	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7835	}
7836
7837
7838	/**
7839	* Worker for loading the guest-state bits in the inner VT-x execution loop.
7840	*
7841	* @param pVM Pointer to the VM.
7842	* @param pVCpu Pointer to the VMCPU.
7843	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7844	* out-of-sync. Make sure to update the required fields
7845	* before using them.
7846	*/
7847	DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7848	{
7849	HMVMX_ASSERT_PREEMPT_SAFE();
7850
7851	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7852	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7853	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7854	#endif
7855
7856	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7857	{
7858	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7859	AssertRC(rc);
7860	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7861	}
7862	else if (HMCPU_CF_VALUE(pVCpu))
7863	{
7864	int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7865	AssertRC(rc);
7866	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7867	}
7868
7869	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7870	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7871	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
7872	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
7873
7874	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7875	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7876	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7877	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7878	#endif
7879	}
7880
7881
7882	/**
7883	* Does the preparations before executing guest code in VT-x.
7884	*
7885	* This may cause longjmps to ring-3 and may even result in rescheduling to the
7886	* recompiler. We must be cautious what we do here regarding committing
7887	* guest-state information into the VMCS assuming we assuredly execute the
7888	* guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7889	* and clearing the common-state (TRPM/forceflags), we must undo those changes
7890	* so that the recompiler can (and should) use them when it resumes guest
7891	* execution. Otherwise such operations must be done when we can no longer
7892	* exit to ring-3.
7893	*
7894	* @returns Strict VBox status code.
7895	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7896	* have been disabled.
7897	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7898	* double-fault into the guest.
7899	* @retval VINF_* scheduling changes, we have to go back to ring-3.
7900	*
7901	* @param pVM Pointer to the VM.
7902	* @param pVCpu Pointer to the VMCPU.
7903	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7904	* out-of-sync. Make sure to update the required fields
7905	* before using them.
7906	* @param pVmxTransient Pointer to the VMX transient structure.
7907	*/
7908	static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7909	{
7910	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7911
7912	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7913	PGMRZDynMapFlushAutoSet(pVCpu);
7914	#endif
7915
7916	/* Check force flag actions that might require us to go back to ring-3. */
7917	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7918	if (rc != VINF_SUCCESS)
7919	return rc;
7920
7921	#ifndef IEM_VERIFICATION_MODE_FULL
7922	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7923	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7924	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7925	{
7926	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7927	RTGCPHYS GCPhysApicBase;
7928	GCPhysApicBase = pMixedCtx->msrApicBase;
7929	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7930
7931	/* Unalias any existing mapping. */
7932	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7933	AssertRCReturn(rc, rc);
7934
7935	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7936	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7937	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
7938	AssertRCReturn(rc, rc);
7939
7940	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7941	}
7942	#endif /* !IEM_VERIFICATION_MODE_FULL */
7943
7944	/* Load the guest state bits, we can handle longjmps/getting preempted here. */
7945	hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7946
7947	/*
7948	* Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7949	* we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7950	*/
7951	if (TRPMHasTrap(pVCpu))
7952	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7953	else if (!pVCpu->hm.s.Event.fPending)
7954	hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7955
7956	/*
7957	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7958	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7959	*/
7960	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7961	if (RT_UNLIKELY(rc != VINF_SUCCESS))
7962	{
7963	Assert(rc == VINF_EM_RESET);
7964	return rc;
7965	}
7966
7967	/*
7968	* No longjmps to ring-3 from this point on!!!
7969	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7970	* This also disables flushing of the R0-logger instance (if any).
7971	*/
7972	VMMRZCallRing3Disable(pVCpu);
7973
7974	/*
7975	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7976	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7977	*
7978	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7979	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7980	*
7981	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7982	* executing guest code.
7983	*/
7984	pVmxTransient->uEflags = ASMIntDisableFlags();
7985	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
7986	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7987	{
7988	hmR0VmxClearEventVmcs(pVCpu);
7989	ASMSetFlags(pVmxTransient->uEflags);
7990	VMMRZCallRing3Enable(pVCpu);
7991	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7992	return VINF_EM_RAW_TO_R3;
7993	}
7994	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7995	{
7996	hmR0VmxClearEventVmcs(pVCpu);
7997	ASMSetFlags(pVmxTransient->uEflags);
7998	VMMRZCallRing3Enable(pVCpu);
7999	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8000	return VINF_EM_RAW_INTERRUPT;
8001	}
8002
8003	/* We've injected any pending events. This is really the point of no return (to ring-3). */
8004	pVCpu->hm.s.Event.fPending = false;
8005
8006	return VINF_SUCCESS;
8007	}
8008
8009
8010	/**
8011	* Prepares to run guest code in VT-x and we've committed to doing so. This
8012	* means there is no backing out to ring-3 or anywhere else at this
8013	* point.
8014	*
8015	* @param pVM Pointer to the VM.
8016	* @param pVCpu Pointer to the VMCPU.
8017	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8018	* out-of-sync. Make sure to update the required fields
8019	* before using them.
8020	* @param pVmxTransient Pointer to the VMX transient structure.
8021	*
8022	* @remarks Called with preemption disabled.
8023	* @remarks No-long-jump zone!!!
8024	*/
8025	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8026	{
8027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8028	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8029	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8030
8031	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8032	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8033
8034	/*
8035	* If we are injecting events to a real-on-v86 mode guest, we may have to update
8036	* RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8037	* Reload only the necessary state, the assertion will catch if other parts of the code
8038	* change.
8039	*/
8040	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8041	{
8042	hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8043	hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8044	}
8045
8046	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8047	if (!CPUMIsGuestFPUStateActive(pVCpu))
8048	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8049	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8050	#endif
8051
8052	if ( pVCpu->hm.s.fUseGuestFpu
8053	&& !CPUMIsGuestFPUStateActive(pVCpu))
8054	{
8055	CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8056	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8057	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8058	}
8059
8060	/*
8061	* The host MSR values the very first time around won't be updated, so we need to
8062	* fill those values in. Subsequently, it's updated as part of the host state.
8063	*/
8064	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8065	&& pVCpu->hm.s.vmx.cMsrs > 0)
8066	{
8067	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8068	}
8069
8070	/*
8071	* Load the host state bits as we may've been preempted (only happens when
8072	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8073	*/
8074	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8075	{
8076	/* This ASSUMES that pfnStartVM has been set up already. */
8077	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8078	AssertRC(rc);
8079	STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8080	}
8081	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8082
8083	/*
8084	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
8085	*/
8086	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8087	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8088	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8089
8090	/* Store status of the shared guest-host state at the time of VM-entry. */
8091	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8092	if (CPUMIsGuestInLongModeEx(pMixedCtx))
8093	{
8094	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8095	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8096	}
8097	else
8098	#endif
8099	{
8100	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8101	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8102	}
8103	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8104
8105	/*
8106	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8107	*/
8108	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8109	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8110
8111	PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8112	RTCPUID idCurrentCpu = pCpu->idCpu;
8113	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8114	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
8115	{
8116	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8117	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8118	}
8119
8120	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8121	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8122	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8123	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8124
8125	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8126
8127	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8128	to start executing. */
8129
8130	/*
8131	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8132	*/
8133	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8134	{
8135	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8136	{
8137	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8138	AssertRC(rc2);
8139	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8140	uint64_t u64GuestTscAuxMsr;
8141	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTscAuxMsr);
8142	AssertRC(rc2);
8143	hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, u64GuestTscAuxMsr, true /* fUpdateHostMsr */);
8144	}
8145	else
8146	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8147	}
8148	#ifdef VBOX_STRICT
8149	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8150	#endif
8151	}
8152
8153
8154	/**
8155	* Performs some essential restoration of state after running guest code in
8156	* VT-x.
8157	*
8158	* @param pVM Pointer to the VM.
8159	* @param pVCpu Pointer to the VMCPU.
8160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8161	* out-of-sync. Make sure to update the required fields
8162	* before using them.
8163	* @param pVmxTransient Pointer to the VMX transient structure.
8164	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8165	*
8166	* @remarks Called with interrupts disabled, and returns with interrups enabled!
8167	*
8168	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
8169	* unconditionally when it is safe to do so.
8170	*/
8171	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8172	{
8173	NOREF(pVM);
8174
8175	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8176
8177	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8178	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8179	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8180	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8181	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8182
8183	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8184	{
8185	/** @todo Find a way to fix hardcoding a guestimate. */
8186	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8187	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8188	}
8189
8190	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8191	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8192	Assert(!(ASMGetFlags() & X86_EFL_IF));
8193	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8194
8195	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8196	if (CPUMIsGuestFPUStateActive(pVCpu))
8197	{
8198	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8199	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8200	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8201	}
8202	#endif
8203
8204	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8205	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8206	ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8207	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8208
8209	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8210	uint32_t uExitReason;
8211	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8212	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8213	AssertRC(rc);
8214	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8215	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8216
8217	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
8218	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8219	{
8220	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8221	pVmxTransient->fVMEntryFailed));
8222	return;
8223	}
8224
8225	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8226	{
8227	/* Update the guest interruptibility-state from the VMCS. */
8228	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8229	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
8230	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8231	AssertRC(rc);
8232	#endif
8233	/*
8234	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8235	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8236	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8237	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8238	*/
8239	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8240	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8241	{
8242	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8243	AssertRC(rc);
8244	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8245	}
8246	}
8247	}
8248
8249
8250
8251	/**
8252	* Runs the guest code using VT-x the normal way.
8253	*
8254	* @returns VBox status code.
8255	* @param pVM Pointer to the VM.
8256	* @param pVCpu Pointer to the VMCPU.
8257	* @param pCtx Pointer to the guest-CPU context.
8258	*
8259	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
8260	*/
8261	static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8262	{
8263	VMXTRANSIENT VmxTransient;
8264	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8265	int rc = VERR_INTERNAL_ERROR_5;
8266	uint32_t cLoops = 0;
8267
8268	for (;; cLoops++)
8269	{
8270	Assert(!HMR0SuspendPending());
8271	HMVMX_ASSERT_CPU_SAFE();
8272
8273	/* Preparatory work for running guest code, this may force us to return
8274	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8275	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8276	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8277	if (rc != VINF_SUCCESS)
8278	break;
8279
8280	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8281	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8282	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8283
8284	/* Restore any residual host-state and save any bits shared between host
8285	and guest into the guest-CPU state. Re-enables interrupts! */
8286	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8287
8288	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8289	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8290	{
8291	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8292	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8293	return rc;
8294	}
8295
8296	/* Handle the VM-exit. */
8297	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8299	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8300	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8301	HMVMX_START_EXIT_DISPATCH_PROF();
8302	#ifdef HMVMX_USE_FUNCTION_TABLE
8303	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8304	#else
8305	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8306	#endif
8307	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8308	if (rc != VINF_SUCCESS)
8309	break;
8310	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8311	{
8312	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8313	rc = VINF_EM_RAW_INTERRUPT;
8314	break;
8315	}
8316	}
8317
8318	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8319	return rc;
8320	}
8321
8322
8323	/**
8324	* Single steps guest code using VT-x.
8325	*
8326	* @returns VBox status code.
8327	* @param pVM Pointer to the VM.
8328	* @param pVCpu Pointer to the VMCPU.
8329	* @param pCtx Pointer to the guest-CPU context.
8330	*
8331	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8332	*/
8333	static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8334	{
8335	VMXTRANSIENT VmxTransient;
8336	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8337	int rc = VERR_INTERNAL_ERROR_5;
8338	uint32_t cLoops = 0;
8339	uint16_t uCsStart = pCtx->cs.Sel;
8340	uint64_t uRipStart = pCtx->rip;
8341
8342	for (;; cLoops++)
8343	{
8344	Assert(!HMR0SuspendPending());
8345	HMVMX_ASSERT_CPU_SAFE();
8346
8347	/* Preparatory work for running guest code, this may force us to return
8348	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8349	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8350	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8351	if (rc != VINF_SUCCESS)
8352	break;
8353
8354	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8355	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8356	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8357
8358	/* Restore any residual host-state and save any bits shared between host
8359	and guest into the guest-CPU state. Re-enables interrupts! */
8360	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8361
8362	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8363	if (RT_UNLIKELY(rc != VINF_SUCCESS))
8364	{
8365	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8366	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8367	return rc;
8368	}
8369
8370	/* Handle the VM-exit. */
8371	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8372	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8373	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8374	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8375	HMVMX_START_EXIT_DISPATCH_PROF();
8376	#ifdef HMVMX_USE_FUNCTION_TABLE
8377	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8378	#else
8379	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8380	#endif
8381	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8382	if (rc != VINF_SUCCESS)
8383	break;
8384	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8385	{
8386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8387	rc = VINF_EM_RAW_INTERRUPT;
8388	break;
8389	}
8390
8391	/*
8392	* Did the RIP change, if so, consider it a single step.
8393	* Otherwise, make sure one of the TFs gets set.
8394	*/
8395	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8396	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8397	AssertRCReturn(rc2, rc2);
8398	if ( pCtx->rip != uRipStart
8399	\|\| pCtx->cs.Sel != uCsStart)
8400	{
8401	rc = VINF_EM_DBG_STEPPED;
8402	break;
8403	}
8404	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8405	}
8406
8407	/*
8408	* Clear the X86_EFL_TF if necessary.
8409	*/
8410	if (pVCpu->hm.s.fClearTrapFlag)
8411	{
8412	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8413	AssertRCReturn(rc2, rc2);
8414	pVCpu->hm.s.fClearTrapFlag = false;
8415	pCtx->eflags.Bits.u1TF = 0;
8416	}
8417	/** @todo there seems to be issues with the resume flag when the monitor trap
8418	* flag is pending without being used. Seen early in bios init when
8419	* accessing APIC page in prot mode. */
8420
8421	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8422	return rc;
8423	}
8424
8425
8426	/**
8427	* Runs the guest code using VT-x.
8428	*
8429	* @returns VBox status code.
8430	* @param pVM Pointer to the VM.
8431	* @param pVCpu Pointer to the VMCPU.
8432	* @param pCtx Pointer to the guest-CPU context.
8433	*/
8434	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8435	{
8436	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8437	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8438	HMVMX_ASSERT_PREEMPT_SAFE();
8439
8440	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8441
8442	int rc;
8443	if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8444	rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8445	else
8446	rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8447
8448	if (rc == VERR_EM_INTERPRETER)
8449	rc = VINF_EM_RAW_EMULATE_INSTR;
8450	else if (rc == VINF_EM_RESET)
8451	rc = VINF_EM_TRIPLE_FAULT;
8452
8453	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8454	if (RT_FAILURE(rc2))
8455	{
8456	pVCpu->hm.s.u32HMError = rc;
8457	rc = rc2;
8458	}
8459	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8460	return rc;
8461	}
8462
8463
8464	#ifndef HMVMX_USE_FUNCTION_TABLE
8465	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8466	{
8467	#ifdef DEBUG_ramshankar
8468	# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8469	# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8470	#endif
8471	int rc;
8472	switch (rcReason)
8473	{
8474	case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); / rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8475	case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); / rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8476	case VMX_EXIT_IO_INSTR: /* SVVMCS(); / rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8477	case VMX_EXIT_CPUID: /* SVVMCS(); / rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8478	case VMX_EXIT_RDTSC: /* SVVMCS(); / rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8479	case VMX_EXIT_RDTSCP: /* SVVMCS(); / rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8480	case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8481	case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); / rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8482	case VMX_EXIT_MOV_CRX: /* SVVMCS(); / rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8483	case VMX_EXIT_EXT_INT: /* SVVMCS(); / rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8484	case VMX_EXIT_INT_WINDOW: /* SVVMCS(); / rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8485	case VMX_EXIT_MWAIT: /* SVVMCS(); / rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8486	case VMX_EXIT_MONITOR: /* SVVMCS(); / rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8487	case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); / rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8488	case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); / rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8489	case VMX_EXIT_RDMSR: /* SVVMCS(); / rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8490	case VMX_EXIT_WRMSR: /* SVVMCS(); / rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8491	case VMX_EXIT_MOV_DRX: /* SVVMCS(); / rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8492	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); / rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8493	case VMX_EXIT_HLT: /* SVVMCS(); / rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8494	case VMX_EXIT_INVD: /* SVVMCS(); / rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8495	case VMX_EXIT_INVLPG: /* SVVMCS(); / rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8496	case VMX_EXIT_RSM: /* SVVMCS(); / rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8497	case VMX_EXIT_MTF: /* SVVMCS(); / rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8498	case VMX_EXIT_PAUSE: /* SVVMCS(); / rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8499	case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8500	case VMX_EXIT_TR_ACCESS: /* SVVMCS(); / rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8501	case VMX_EXIT_WBINVD: /* SVVMCS(); / rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8502	case VMX_EXIT_XSETBV: /* SVVMCS(); / rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8503	case VMX_EXIT_RDRAND: /* SVVMCS(); / rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8504	case VMX_EXIT_INVPCID: /* SVVMCS(); / rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8505	case VMX_EXIT_GETSEC: /* SVVMCS(); / rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8506	case VMX_EXIT_RDPMC: /* SVVMCS(); / rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); / LDVMCS(); */ break;
8507
8508	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8509	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8510	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8511	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8512	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8513	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8514	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8515	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8516	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8517
8518	case VMX_EXIT_VMCALL:
8519	case VMX_EXIT_VMCLEAR:
8520	case VMX_EXIT_VMLAUNCH:
8521	case VMX_EXIT_VMPTRLD:
8522	case VMX_EXIT_VMPTRST:
8523	case VMX_EXIT_VMREAD:
8524	case VMX_EXIT_VMRESUME:
8525	case VMX_EXIT_VMWRITE:
8526	case VMX_EXIT_VMXOFF:
8527	case VMX_EXIT_VMXON:
8528	case VMX_EXIT_INVEPT:
8529	case VMX_EXIT_INVVPID:
8530	case VMX_EXIT_VMFUNC:
8531	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8532	break;
8533	default:
8534	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8535	break;
8536	}
8537	return rc;
8538	}
8539	#endif
8540
8541	#ifdef DEBUG
8542	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8543	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8544	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8545
8546	# define HMVMX_ASSERT_PREEMPT_CPUID() \
8547	do \
8548	{ \
8549	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8550	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8551	} while (0)
8552
8553	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8554	do { \
8555	AssertPtr(pVCpu); \
8556	AssertPtr(pMixedCtx); \
8557	AssertPtr(pVmxTransient); \
8558	Assert(pVmxTransient->fVMEntryFailed == false); \
8559	Assert(ASMIntAreEnabled()); \
8560	HMVMX_ASSERT_PREEMPT_SAFE(); \
8561	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8562	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8563	HMVMX_ASSERT_PREEMPT_SAFE(); \
8564	if (VMMR0IsLogFlushDisabled(pVCpu)) \
8565	HMVMX_ASSERT_PREEMPT_CPUID(); \
8566	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8567	} while (0)
8568
8569	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8570	do { \
8571	Log4Func(("\n")); \
8572	} while (0)
8573	#else /* Release builds */
8574	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8575	do { \
8576	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8577	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8578	} while (0)
8579	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8580	#endif
8581
8582
8583	/**
8584	* Advances the guest RIP after reading it from the VMCS.
8585	*
8586	* @returns VBox status code.
8587	* @param pVCpu Pointer to the VMCPU.
8588	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8589	* out-of-sync. Make sure to update the required fields
8590	* before using them.
8591	* @param pVmxTransient Pointer to the VMX transient structure.
8592	*
8593	* @remarks No-long-jump zone!!!
8594	*/
8595	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8596	{
8597	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8598	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8599	AssertRCReturn(rc, rc);
8600
8601	pMixedCtx->rip += pVmxTransient->cbInstr;
8602	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8603	return rc;
8604	}
8605
8606
8607	/**
8608	* Tries to determine what part of the guest-state VT-x has deemed as invalid
8609	* and update error record fields accordingly.
8610	*
8611	* @return VMX_IGS_* return codes.
8612	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8613	* wrong with the guest state.
8614	*
8615	* @param pVM Pointer to the VM.
8616	* @param pVCpu Pointer to the VMCPU.
8617	* @param pCtx Pointer to the guest-CPU state.
8618	*/
8619	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8620	{
8621	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8622	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8623	uError = (err); \
8624	break; \
8625	} else do { } while (0)
8626
8627	int rc;
8628	uint32_t uError = VMX_IGS_ERROR;
8629	uint32_t u32Val;
8630	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8631
8632	do
8633	{
8634	/*
8635	* CR0.
8636	*/
8637	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8638	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8639	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8640	See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8641	if (fUnrestrictedGuest)
8642	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
8643
8644	uint32_t u32GuestCR0;
8645	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8646	AssertRCBreak(rc);
8647	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8648	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8649	if ( !fUnrestrictedGuest
8650	&& (u32GuestCR0 & X86_CR0_PG)
8651	&& !(u32GuestCR0 & X86_CR0_PE))
8652	{
8653	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8654	}
8655
8656	/*
8657	* CR4.
8658	*/
8659	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8660	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8661
8662	uint32_t u32GuestCR4;
8663	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8664	AssertRCBreak(rc);
8665	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8666	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8667
8668	/*
8669	* IA32_DEBUGCTL MSR.
8670	*/
8671	uint64_t u64Val;
8672	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8673	AssertRCBreak(rc);
8674	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8675	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8676	{
8677	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8678	}
8679	uint64_t u64DebugCtlMsr = u64Val;
8680
8681	#ifdef VBOX_STRICT
8682	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8683	AssertRCBreak(rc);
8684	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
8685	#endif
8686	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8687
8688	/*
8689	* RIP and RFLAGS.
8690	*/
8691	uint32_t u32Eflags;
8692	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8693	if (HMVMX_IS_64BIT_HOST_MODE())
8694	{
8695	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8696	AssertRCBreak(rc);
8697	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8698	if ( !fLongModeGuest
8699	\|\| !pCtx->cs.Attr.n.u1Long)
8700	{
8701	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8702	}
8703	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8704	* must be identical if the "IA32e mode guest" VM-entry control is 1
8705	* and CS.L is 1. No check applies if the CPU supports 64
8706	* linear-address bits. */
8707
8708	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8709	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8710	AssertRCBreak(rc);
8711	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8712	VMX_IGS_RFLAGS_RESERVED);
8713	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8714	u32Eflags = u64Val;
8715	}
8716	else
8717	#endif
8718	{
8719	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8720	AssertRCBreak(rc);
8721	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8722	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8723	}
8724
8725	if ( fLongModeGuest
8726	\|\| ( fUnrestrictedGuest
8727	&& !(u32GuestCR0 & X86_CR0_PE)))
8728	{
8729	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8730	}
8731
8732	uint32_t u32EntryInfo;
8733	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8734	AssertRCBreak(rc);
8735	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
8736	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8737	{
8738	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8739	}
8740
8741	/*
8742	* 64-bit checks.
8743	*/
8744	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8745	if (HMVMX_IS_64BIT_HOST_MODE())
8746	{
8747	if ( fLongModeGuest
8748	&& !fUnrestrictedGuest)
8749	{
8750	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8751	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8752	}
8753
8754	if ( !fLongModeGuest
8755	&& (u32GuestCR4 & X86_CR4_PCIDE))
8756	{
8757	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8758	}
8759
8760	/** @todo CR3 field must be such that bits 63:52 and bits in the range
8761	* 51:32 beyond the processor's physical-address width are 0. */
8762
8763	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8764	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8765	{
8766	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8767	}
8768
8769	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8770	AssertRCBreak(rc);
8771	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8772
8773	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8774	AssertRCBreak(rc);
8775	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8776	}
8777	#endif
8778
8779	/*
8780	* PERF_GLOBAL MSR.
8781	*/
8782	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8783	{
8784	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8785	AssertRCBreak(rc);
8786	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8787	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8788	}
8789
8790	/*
8791	* PAT MSR.
8792	*/
8793	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8794	{
8795	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8796	AssertRCBreak(rc);
8797	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8798	for (unsigned i = 0; i < 8; i++)
8799	{
8800	uint8_t u8Val = (u64Val & 0x7);
8801	if ( u8Val != 0 /* UC */
8802	\|\| u8Val != 1 /* WC */
8803	\|\| u8Val != 4 /* WT */
8804	\|\| u8Val != 5 /* WP */
8805	\|\| u8Val != 6 /* WB */
8806	\|\| u8Val != 7 /* UC- */)
8807	{
8808	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8809	}
8810	u64Val >>= 3;
8811	}
8812	}
8813
8814	/*
8815	* EFER MSR.
8816	*/
8817	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8818	{
8819	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8820	AssertRCBreak(rc);
8821	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8822	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8823	HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8824	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8825	HMVMX_CHECK_BREAK( fUnrestrictedGuest
8826	\|\| (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8827	}
8828
8829	/*
8830	* Segment registers.
8831	*/
8832	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8833	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8834	if (!(u32Eflags & X86_EFL_VM))
8835	{
8836	/* CS */
8837	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8838	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8839	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8840	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8841	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8842	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8843	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8844	/* CS cannot be loaded with NULL in protected mode. */
8845	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8846	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8847	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
8848	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8849	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
8850	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8851	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8852	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8853	else
8854	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8855
8856	/* SS */
8857	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8858	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8859	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8860	if ( !(pCtx->cr0 & X86_CR0_PE)
8861	\|\| pCtx->cs.Attr.n.u4Type == 3)
8862	{
8863	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8864	}
8865	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8866	{
8867	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8868	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8869	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8870	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8871	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8872	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8873	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8874	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8875	}
8876
8877	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8878	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8879	{
8880	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8881	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8882	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8883	\|\| pCtx->ds.Attr.n.u4Type > 11
8884	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8885	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8886	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8887	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8888	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8889	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8890	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8891	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8892	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8893	}
8894	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8895	{
8896	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8897	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8898	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8899	\|\| pCtx->es.Attr.n.u4Type > 11
8900	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8901	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8902	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8903	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8904	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8905	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8906	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8907	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8908	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8909	}
8910	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8911	{
8912	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8913	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8914	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8915	\|\| pCtx->fs.Attr.n.u4Type > 11
8916	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8917	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8918	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8919	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8920	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8921	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8922	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8923	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8924	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8925	}
8926	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8927	{
8928	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8929	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8930	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8931	\|\| pCtx->gs.Attr.n.u4Type > 11
8932	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8933	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8934	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8935	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8936	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8937	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8938	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8939	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8940	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8941	}
8942	/* 64-bit capable CPUs. */
8943	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8944	if (HMVMX_IS_64BIT_HOST_MODE())
8945	{
8946	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8947	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8948	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8949	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8950	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8951	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
8952	VMX_IGS_LONGMODE_SS_BASE_INVALID);
8953	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
8954	VMX_IGS_LONGMODE_DS_BASE_INVALID);
8955	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
8956	VMX_IGS_LONGMODE_ES_BASE_INVALID);
8957	}
8958	#endif
8959	}
8960	else
8961	{
8962	/* V86 mode checks. */
8963	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8964	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8965	{
8966	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8967	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8968	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8969	}
8970	else
8971	{
8972	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8973	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8974	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8975	}
8976
8977	/* CS */
8978	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8979	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8980	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8981	/* SS */
8982	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8983	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8984	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8985	/* DS */
8986	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8987	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8988	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8989	/* ES */
8990	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8991	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8992	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8993	/* FS */
8994	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8995	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8996	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8997	/* GS */
8998	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8999	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9000	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9001	/* 64-bit capable CPUs. */
9002	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9003	if (HMVMX_IS_64BIT_HOST_MODE())
9004	{
9005	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9006	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9007	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9008	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9009	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9010	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
9011	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9012	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
9013	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9014	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
9015	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9016	}
9017	#endif
9018	}
9019
9020	/*
9021	* TR.
9022	*/
9023	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9024	/* 64-bit capable CPUs. */
9025	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9026	if (HMVMX_IS_64BIT_HOST_MODE())
9027	{
9028	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9029	}
9030	#endif
9031	if (fLongModeGuest)
9032	{
9033	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9034	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9035	}
9036	else
9037	{
9038	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9039	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9040	VMX_IGS_TR_ATTR_TYPE_INVALID);
9041	}
9042	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9043	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9044	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9045	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9046	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9047	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9048	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9049	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9050
9051	/*
9052	* GDTR and IDTR.
9053	*/
9054	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9055	if (HMVMX_IS_64BIT_HOST_MODE())
9056	{
9057	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9058	AssertRCBreak(rc);
9059	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9060
9061	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9062	AssertRCBreak(rc);
9063	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9064	}
9065	#endif
9066
9067	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9068	AssertRCBreak(rc);
9069	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9070
9071	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9072	AssertRCBreak(rc);
9073	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9074
9075	/*
9076	* Guest Non-Register State.
9077	*/
9078	/* Activity State. */
9079	uint32_t u32ActivityState;
9080	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9081	AssertRCBreak(rc);
9082	HMVMX_CHECK_BREAK( !u32ActivityState
9083	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9084	VMX_IGS_ACTIVITY_STATE_INVALID);
9085	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9086	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9087	uint32_t u32IntrState;
9088	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9089	AssertRCBreak(rc);
9090	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9091	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9092	{
9093	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9094	}
9095
9096	/** @todo Activity state and injecting interrupts. Left as a todo since we
9097	* currently don't use activity states but ACTIVE. */
9098
9099	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9100	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9101
9102	/* Guest interruptibility-state. */
9103	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9104	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9105	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9106	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9107	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9108	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9109	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9110	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9111	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9112	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9113	{
9114	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9115	{
9116	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9117	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9118	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9119	}
9120	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9121	{
9122	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9123	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9124	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9125	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9126	}
9127	}
9128	/** @todo Assumes the processor is not in SMM. */
9129	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9130	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9131	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9132	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9133	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9134	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9135	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9136	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9137	{
9138	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9139	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9140	}
9141
9142	/* Pending debug exceptions. */
9143	if (HMVMX_IS_64BIT_HOST_MODE())
9144	{
9145	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9146	AssertRCBreak(rc);
9147	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9148	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9149	u32Val = u64Val; /* For pending debug exceptions checks below. */
9150	}
9151	else
9152	{
9153	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9154	AssertRCBreak(rc);
9155	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9156	HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9157	}
9158
9159	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9160	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9161	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9162	{
9163	if ( (u32Eflags & X86_EFL_TF)
9164	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9165	{
9166	/* Bit 14 is PendingDebug.BS. */
9167	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9168	}
9169	if ( !(u32Eflags & X86_EFL_TF)
9170	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9171	{
9172	/* Bit 14 is PendingDebug.BS. */
9173	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9174	}
9175	}
9176
9177	/* VMCS link pointer. */
9178	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9179	AssertRCBreak(rc);
9180	if (u64Val != UINT64_C(0xffffffffffffffff))
9181	{
9182	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9183	/** @todo Bits beyond the processor's physical-address width MBZ. */
9184	/** @todo 32-bit located in memory referenced by value of this field (as a
9185	* physical address) must contain the processor's VMCS revision ID. */
9186	/** @todo SMM checks. */
9187	}
9188
9189	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9190	* not using Nested Paging? */
9191	if ( pVM->hm.s.fNestedPaging
9192	&& !fLongModeGuest
9193	&& CPUMIsGuestInPAEModeEx(pCtx))
9194	{
9195	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9196	AssertRCBreak(rc);
9197	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9198
9199	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9200	AssertRCBreak(rc);
9201	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9202
9203	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9204	AssertRCBreak(rc);
9205	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9206
9207	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9208	AssertRCBreak(rc);
9209	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9210	}
9211
9212	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9213	if (uError == VMX_IGS_ERROR)
9214	uError = VMX_IGS_REASON_NOT_FOUND;
9215	} while (0);
9216
9217	pVCpu->hm.s.u32HMError = uError;
9218	return uError;
9219
9220	#undef HMVMX_ERROR_BREAK
9221	#undef HMVMX_CHECK_BREAK
9222	}
9223
9224	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9225	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9226	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9227
9228	/** @name VM-exit handlers.
9229	* @{
9230	*/
9231
9232	/**
9233	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9234	*/
9235	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9236	{
9237	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9238	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9239	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9240	if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9241	return VINF_SUCCESS;
9242	return VINF_EM_RAW_INTERRUPT;
9243	}
9244
9245
9246	/**
9247	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9248	*/
9249	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9250	{
9251	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9252	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9253
9254	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9255	AssertRCReturn(rc, rc);
9256
9257	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9258	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9259	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9260	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9261
9262	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9263	{
9264	/*
9265	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9266	* anything we inject is not going to cause a VM-exit directly for the event being injected.
9267	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9268	*
9269	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9270	*/
9271	VMXDispatchHostNmi();
9272	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9273	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9274	return VINF_SUCCESS;
9275	}
9276
9277	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9278	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9279	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9280	{
9281	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9282	return VINF_SUCCESS;
9283	}
9284	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9285	{
9286	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9287	return rc;
9288	}
9289
9290	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9291	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9292	switch (uIntType)
9293	{
9294	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9295	Assert(uVector == X86_XCPT_DB);
9296	/* no break */
9297	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9298	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9299	/* no break */
9300	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9301	{
9302	switch (uVector)
9303	{
9304	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9305	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9306	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9307	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9308	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9309	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9310	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9311	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9312	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9313	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9314	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9315	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9316	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9317	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9318	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9319	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9320	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9321	#endif
9322	default:
9323	{
9324	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9325	AssertRCReturn(rc, rc);
9326
9327	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9328	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9329	{
9330	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9331	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9332	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9333
9334	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9335	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9336	AssertRCReturn(rc, rc);
9337	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9338	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9339	0 /* GCPtrFaultAddress */);
9340	AssertRCReturn(rc, rc);
9341	}
9342	else
9343	{
9344	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9345	pVCpu->hm.s.u32HMError = uVector;
9346	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9347	}
9348	break;
9349	}
9350	}
9351	break;
9352	}
9353
9354	default:
9355	{
9356	pVCpu->hm.s.u32HMError = uExitIntInfo;
9357	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9358	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9359	break;
9360	}
9361	}
9362	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9363	return rc;
9364	}
9365
9366
9367	/**
9368	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9369	*/
9370	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9371	{
9372	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9373
9374	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9375	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
9376	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
9377	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9378	AssertRCReturn(rc, rc);
9379
9380	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9382	return VINF_SUCCESS;
9383	}
9384
9385
9386	/**
9387	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9388	*/
9389	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9390	{
9391	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9392	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9393	HMVMX_RETURN_UNEXPECTED_EXIT();
9394	}
9395
9396
9397	/**
9398	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9399	*/
9400	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9401	{
9402	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9403	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9404	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9405	}
9406
9407
9408	/**
9409	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9410	*/
9411	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9412	{
9413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9414	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9415	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9416	}
9417
9418
9419	/**
9420	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9421	*/
9422	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9423	{
9424	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9425	PVM pVM = pVCpu->CTX_SUFF(pVM);
9426	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9427	if (RT_LIKELY(rc == VINF_SUCCESS))
9428	{
9429	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9430	Assert(pVmxTransient->cbInstr == 2);
9431	}
9432	else
9433	{
9434	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9435	rc = VERR_EM_INTERPRETER;
9436	}
9437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9438	return rc;
9439	}
9440
9441
9442	/**
9443	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9444	*/
9445	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9446	{
9447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9448	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9449	AssertRCReturn(rc, rc);
9450
9451	if (pMixedCtx->cr4 & X86_CR4_SMXE)
9452	return VINF_EM_RAW_EMULATE_INSTR;
9453
9454	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9455	HMVMX_RETURN_UNEXPECTED_EXIT();
9456	}
9457
9458
9459	/**
9460	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9461	*/
9462	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9463	{
9464	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9465	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9466	AssertRCReturn(rc, rc);
9467
9468	PVM pVM = pVCpu->CTX_SUFF(pVM);
9469	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9470	if (RT_LIKELY(rc == VINF_SUCCESS))
9471	{
9472	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9473	Assert(pVmxTransient->cbInstr == 2);
9474	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9475	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9476	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9477	}
9478	else
9479	{
9480	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9481	rc = VERR_EM_INTERPRETER;
9482	}
9483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9484	return rc;
9485	}
9486
9487
9488	/**
9489	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9490	*/
9491	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9492	{
9493	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9494	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9495	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9496	AssertRCReturn(rc, rc);
9497
9498	PVM pVM = pVCpu->CTX_SUFF(pVM);
9499	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9500	if (RT_LIKELY(rc == VINF_SUCCESS))
9501	{
9502	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9503	Assert(pVmxTransient->cbInstr == 3);
9504	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9505	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9506	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9507	}
9508	else
9509	{
9510	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9511	rc = VERR_EM_INTERPRETER;
9512	}
9513	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9514	return rc;
9515	}
9516
9517
9518	/**
9519	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9520	*/
9521	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9522	{
9523	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9524	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9525	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9526	AssertRCReturn(rc, rc);
9527
9528	PVM pVM = pVCpu->CTX_SUFF(pVM);
9529	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9530	if (RT_LIKELY(rc == VINF_SUCCESS))
9531	{
9532	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9533	Assert(pVmxTransient->cbInstr == 2);
9534	}
9535	else
9536	{
9537	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9538	rc = VERR_EM_INTERPRETER;
9539	}
9540	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9541	return rc;
9542	}
9543
9544
9545	/**
9546	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9547	*/
9548	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9549	{
9550	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9551	PVM pVM = pVCpu->CTX_SUFF(pVM);
9552	Assert(!pVM->hm.s.fNestedPaging);
9553
9554	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9555	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9556	AssertRCReturn(rc, rc);
9557
9558	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9559	rc = VBOXSTRICTRC_VAL(rc2);
9560	if (RT_LIKELY(rc == VINF_SUCCESS))
9561	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9562	else
9563	{
9564	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9565	pVmxTransient->uExitQualification, rc));
9566	}
9567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9568	return rc;
9569	}
9570
9571
9572	/**
9573	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9574	*/
9575	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9576	{
9577	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9578	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9579	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9580	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9581	AssertRCReturn(rc, rc);
9582
9583	PVM pVM = pVCpu->CTX_SUFF(pVM);
9584	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9585	if (RT_LIKELY(rc == VINF_SUCCESS))
9586	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9587	else
9588	{
9589	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9590	rc = VERR_EM_INTERPRETER;
9591	}
9592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9593	return rc;
9594	}
9595
9596
9597	/**
9598	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9599	*/
9600	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9601	{
9602	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9603	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9604	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9605	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9606	AssertRCReturn(rc, rc);
9607
9608	PVM pVM = pVCpu->CTX_SUFF(pVM);
9609	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9610	rc = VBOXSTRICTRC_VAL(rc2);
9611	if (RT_LIKELY( rc == VINF_SUCCESS
9612	\|\| rc == VINF_EM_HALT))
9613	{
9614	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9615	AssertRCReturn(rc3, rc3);
9616
9617	if ( rc == VINF_EM_HALT
9618	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9619	{
9620	rc = VINF_SUCCESS;
9621	}
9622	}
9623	else
9624	{
9625	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9626	rc = VERR_EM_INTERPRETER;
9627	}
9628	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
9629	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9631	return rc;
9632	}
9633
9634
9635	/**
9636	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9637	*/
9638	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9639	{
9640	/*
9641	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9642	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9643	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
9644	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9645	*/
9646	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9647	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9648	HMVMX_RETURN_UNEXPECTED_EXIT();
9649	}
9650
9651
9652	/**
9653	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9654	*/
9655	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9656	{
9657	/*
9658	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9659	* root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9660	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
9661	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9662	*/
9663	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9664	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9665	HMVMX_RETURN_UNEXPECTED_EXIT();
9666	}
9667
9668
9669	/**
9670	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9671	*/
9672	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9673	{
9674	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9675	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9676	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9677	HMVMX_RETURN_UNEXPECTED_EXIT();
9678	}
9679
9680
9681	/**
9682	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9683	*/
9684	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9685	{
9686	/*
9687	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9688	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9689	* See Intel spec. 25.3 "Other Causes of VM-exits".
9690	*/
9691	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9692	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9693	HMVMX_RETURN_UNEXPECTED_EXIT();
9694	}
9695
9696
9697	/**
9698	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9699	* VM-exit.
9700	*/
9701	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9702	{
9703	/*
9704	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9705	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9706	*
9707	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9708	* See Intel spec. "23.8 Restrictions on VMX operation".
9709	*/
9710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9711	return VINF_SUCCESS;
9712	}
9713
9714
9715	/**
9716	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9717	* VM-exit.
9718	*/
9719	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9720	{
9721	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9722	return VINF_EM_RESET;
9723	}
9724
9725
9726	/**
9727	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9728	*/
9729	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9730	{
9731	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9732	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9733	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9734	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9735	AssertRCReturn(rc, rc);
9736
9737	pMixedCtx->rip++;
9738	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9739	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9740	rc = VINF_SUCCESS;
9741	else
9742	rc = VINF_EM_HALT;
9743
9744	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9745	return rc;
9746	}
9747
9748
9749	/**
9750	* VM-exit handler for instructions that result in a #UD exception delivered to
9751	* the guest.
9752	*/
9753	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9754	{
9755	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9756	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9757	return VINF_SUCCESS;
9758	}
9759
9760
9761	/**
9762	* VM-exit handler for expiry of the VMX preemption timer.
9763	*/
9764	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9765	{
9766	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9767
9768	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9769	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9770
9771	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9772	PVM pVM = pVCpu->CTX_SUFF(pVM);
9773	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9775	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9776	}
9777
9778
9779	/**
9780	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9781	*/
9782	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9783	{
9784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9785
9786	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9787	/** @todo check if XSETBV is supported by the recompiler. */
9788	return VERR_EM_INTERPRETER;
9789	}
9790
9791
9792	/**
9793	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9794	*/
9795	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9796	{
9797	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9798
9799	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9800	/** @todo implement EMInterpretInvpcid() */
9801	return VERR_EM_INTERPRETER;
9802	}
9803
9804
9805	/**
9806	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9807	* Error VM-exit.
9808	*/
9809	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9810	{
9811	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9812	AssertRCReturn(rc, rc);
9813
9814	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9815	NOREF(uInvalidReason);
9816
9817	#ifdef VBOX_STRICT
9818	uint32_t uIntrState;
9819	HMVMXHCUINTREG uHCReg;
9820	uint64_t u64Val;
9821	uint32_t u32Val;
9822
9823	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9824	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9825	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
9826	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9827	AssertRCReturn(rc, rc);
9828
9829	Log4(("uInvalidReason %u\n", uInvalidReason));
9830	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
9831	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9832	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9833	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9834
9835	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9836	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9837	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9838	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9839	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9840	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9841	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9842	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9843	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9844	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9845	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9846	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9847	#else
9848	NOREF(pVmxTransient);
9849	#endif
9850
9851	HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9852	return VERR_VMX_INVALID_GUEST_STATE;
9853	}
9854
9855
9856	/**
9857	* VM-exit handler for VM-entry failure due to an MSR-load
9858	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9859	*/
9860	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9861	{
9862	NOREF(pVmxTransient);
9863	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9864	HMVMX_RETURN_UNEXPECTED_EXIT();
9865	}
9866
9867
9868	/**
9869	* VM-exit handler for VM-entry failure due to a machine-check event
9870	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9871	*/
9872	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9873	{
9874	NOREF(pVmxTransient);
9875	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
9876	HMVMX_RETURN_UNEXPECTED_EXIT();
9877	}
9878
9879
9880	/**
9881	* VM-exit handler for all undefined reasons. Should never ever happen.. in
9882	* theory.
9883	*/
9884	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9885	{
9886	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9887	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
9888	return VERR_VMX_UNDEFINED_EXIT_CODE;
9889	}
9890
9891
9892	/**
9893	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9894	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9895	* Conditional VM-exit.
9896	*/
9897	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9898	{
9899	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9900
9901	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9903	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9904	return VERR_EM_INTERPRETER;
9905	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9906	HMVMX_RETURN_UNEXPECTED_EXIT();
9907	}
9908
9909
9910	/**
9911	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9912	*/
9913	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9914	{
9915	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9916
9917	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9918	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9919	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9920	return VERR_EM_INTERPRETER;
9921	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9922	HMVMX_RETURN_UNEXPECTED_EXIT();
9923	}
9924
9925
9926	/**
9927	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9928	*/
9929	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9930	{
9931	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9932
9933	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9934	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9935	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9936	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9937	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9938	{
9939	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9940	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9941	}
9942	AssertRCReturn(rc, rc);
9943	Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9944
9945	#ifdef VBOX_STRICT
9946	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
9947	{
9948	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
9949	{
9950	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9951	HMVMX_RETURN_UNEXPECTED_EXIT();
9952	}
9953	# if HC_ARCH_BITS == 64
9954	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
9955	&& hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
9956	{
9957	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
9958	HMVMX_RETURN_UNEXPECTED_EXIT();
9959	}
9960	# endif
9961	}
9962	#endif
9963
9964	PVM pVM = pVCpu->CTX_SUFF(pVM);
9965	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9966	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
9967	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9968	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9969
9970	if (RT_LIKELY(rc == VINF_SUCCESS))
9971	{
9972	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9973	Assert(pVmxTransient->cbInstr == 2);
9974	}
9975	return rc;
9976	}
9977
9978
9979	/**
9980	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9981	*/
9982	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9983	{
9984	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9985	PVM pVM = pVCpu->CTX_SUFF(pVM);
9986	int rc = VINF_SUCCESS;
9987
9988	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9989	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9990	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9991	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9992	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9993	{
9994	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
9995	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9996	}
9997	AssertRCReturn(rc, rc);
9998	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9999
10000	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10001	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10002	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10003
10004	if (RT_LIKELY(rc == VINF_SUCCESS))
10005	{
10006	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10007
10008	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
10009	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10010	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10011	{
10012	/* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10013	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10014	EMInterpretWrmsr() changes it. */
10015	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10016	}
10017	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10018	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10019
10020	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10021	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10022	{
10023	switch (pMixedCtx->ecx)
10024	{
10025	case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10026	case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10027	case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10028	case MSR_K8_FS_BASE: /* no break */
10029	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10030	default:
10031	{
10032	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10033	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10034	#if HC_ARCH_BITS == 64
10035	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10036	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10037	#endif
10038	break;
10039	}
10040	}
10041	}
10042	#ifdef VBOX_STRICT
10043	else
10044	{
10045	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10046	switch (pMixedCtx->ecx)
10047	{
10048	case MSR_IA32_SYSENTER_CS:
10049	case MSR_IA32_SYSENTER_EIP:
10050	case MSR_IA32_SYSENTER_ESP:
10051	case MSR_K8_FS_BASE:
10052	case MSR_K8_GS_BASE:
10053	{
10054	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10055	HMVMX_RETURN_UNEXPECTED_EXIT();
10056	}
10057
10058	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10059	default:
10060	{
10061	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10062	{
10063	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10064	pMixedCtx->ecx));
10065	HMVMX_RETURN_UNEXPECTED_EXIT();
10066	}
10067
10068	#if HC_ARCH_BITS == 64
10069	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10070	{
10071	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10072	HMVMX_RETURN_UNEXPECTED_EXIT();
10073	}
10074	#endif
10075	break;
10076	}
10077	}
10078	}
10079	#endif /* VBOX_STRICT */
10080	}
10081	return rc;
10082	}
10083
10084
10085	/**
10086	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10087	*/
10088	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10089	{
10090	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10091
10092	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10093	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10094	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10095	return VERR_EM_INTERPRETER;
10096	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10097	HMVMX_RETURN_UNEXPECTED_EXIT();
10098	}
10099
10100
10101	/**
10102	* VM-exit handler for when the TPR value is lowered below the specified
10103	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10104	*/
10105	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10106	{
10107	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10108	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10109
10110	/*
10111	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10112	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10113	* resume guest execution.
10114	*/
10115	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10117	return VINF_SUCCESS;
10118	}
10119
10120
10121	/**
10122	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10123	* VM-exit.
10124	*
10125	* @retval VINF_SUCCESS when guest execution can continue.
10126	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10127	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10128	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10129	* recompiler.
10130	*/
10131	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10132	{
10133	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10134	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10135	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10136	AssertRCReturn(rc, rc);
10137
10138	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10139	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10140	PVM pVM = pVCpu->CTX_SUFF(pVM);
10141	switch (uAccessType)
10142	{
10143	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10144	{
10145	#if 0
10146	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10147	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10148	#else
10149	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10150	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10151	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10152	#endif
10153	AssertRCReturn(rc, rc);
10154
10155	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10156	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10157	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10158	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
10159
10160	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10161	{
10162	case 0: /* CR0 */
10163	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10164	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10165	break;
10166	case 2: /* CR2 */
10167	/* Nothing to do here, CR2 it's not part of the VMCS. */
10168	break;
10169	case 3: /* CR3 */
10170	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10171	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10172	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10173	break;
10174	case 4: /* CR4 */
10175	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10176	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10177	break;
10178	case 8: /* CR8 */
10179	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10180	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10181	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10182	break;
10183	default:
10184	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10185	break;
10186	}
10187
10188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10189	break;
10190	}
10191
10192	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10193	{
10194	/* EMInterpretCRxRead() requires EFER MSR, CS. */
10195	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10196	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10197	AssertRCReturn(rc, rc);
10198	Assert( !pVM->hm.s.fNestedPaging
10199	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10200	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10201
10202	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10203	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10204	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10205
10206	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10207	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10208	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10209	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10210	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10211	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10212	break;
10213	}
10214
10215	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10216	{
10217	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10218	AssertRCReturn(rc, rc);
10219	rc = EMInterpretCLTS(pVM, pVCpu);
10220	AssertRCReturn(rc, rc);
10221	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10222	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10223	Log4(("CRX CLTS write rc=%d\n", rc));
10224	break;
10225	}
10226
10227	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10228	{
10229	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10230	AssertRCReturn(rc, rc);
10231	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10232	if (RT_LIKELY(rc == VINF_SUCCESS))
10233	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10235	Log4(("CRX LMSW write rc=%d\n", rc));
10236	break;
10237	}
10238
10239	default:
10240	{
10241	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10242	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10243	}
10244	}
10245
10246	/* Validate possible error codes. */
10247	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
10248	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10249	if (RT_SUCCESS(rc))
10250	{
10251	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10252	AssertRCReturn(rc2, rc2);
10253	}
10254
10255	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10256	return rc;
10257	}
10258
10259
10260	/**
10261	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10262	* VM-exit.
10263	*/
10264	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10265	{
10266	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10267	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10268
10269	int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10270	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10271	rc2 \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10272	rc2 \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10273	rc2 \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10274	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10275	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10276	AssertRCReturn(rc2, rc2);
10277
10278	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10279	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10280	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10281	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10282	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10283	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10284	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10285
10286	/* I/O operation lookup arrays. */
10287	static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10288	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10289
10290	VBOXSTRICTRC rcStrict;
10291	const uint32_t cbValue = s_aIOSizes[uIOWidth];
10292	const uint32_t cbInstr = pVmxTransient->cbInstr;
10293	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10294	PVM pVM = pVCpu->CTX_SUFF(pVM);
10295	if (fIOString)
10296	{
10297	#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10298	/*
10299	* INS/OUTS - I/O String instruction.
10300	*
10301	* Use instruction-information if available, otherwise fall back on
10302	* interpreting the instruction.
10303	*/
10304	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10305	AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10306	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10307	{
10308	rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10309	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10310	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10311	AssertRCReturn(rc2, rc2);
10312	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10313	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10314	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10315	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10316	if (fIOWrite)
10317	{
10318	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10319	pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10320	}
10321	else
10322	{
10323	/*
10324	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10325	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10326	* See Intel Instruction spec. for "INS".
10327	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10328	*/
10329	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10330	}
10331	}
10332	else
10333	{
10334	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10335	rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10336	AssertRCReturn(rc2, rc2);
10337	rcStrict = IEMExecOne(pVCpu);
10338	}
10339	/** @todo IEM needs to be setting these flags somehow. */
10340	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10341	fUpdateRipAlready = true;
10342	#else
10343	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10344	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10345	if (RT_SUCCESS(rcStrict))
10346	{
10347	if (fIOWrite)
10348	{
10349	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10350	(DISCPUMODE)pDis->uAddrMode, cbValue);
10351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10352	}
10353	else
10354	{
10355	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10356	(DISCPUMODE)pDis->uAddrMode, cbValue);
10357	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10358	}
10359	}
10360	else
10361	{
10362	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10363	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10364	}
10365	#endif
10366	}
10367	else
10368	{
10369	/*
10370	* IN/OUT - I/O instruction.
10371	*/
10372	Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10373	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10374	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10375	if (fIOWrite)
10376	{
10377	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10378	if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10379	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10381	}
10382	else
10383	{
10384	uint32_t u32Result = 0;
10385	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10386	if (IOM_SUCCESS(rcStrict))
10387	{
10388	/* Save result of I/O IN instr. in AL/AX/EAX. */
10389	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
10390	}
10391	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10392	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10394	}
10395	}
10396
10397	if (IOM_SUCCESS(rcStrict))
10398	{
10399	if (!fUpdateRipAlready)
10400	{
10401	pMixedCtx->rip += cbInstr;
10402	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10403	}
10404
10405	/*
10406	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10407	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10408	*/
10409	if (fIOString)
10410	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10411
10412	/*
10413	* If any I/O breakpoints are armed, we need to check if one triggered
10414	* and take appropriate action.
10415	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10416	*/
10417	rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10418	AssertRCReturn(rc2, rc2);
10419
10420	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10421	* execution engines about whether hyper BPs and such are pending. */
10422	uint32_t const uDr7 = pMixedCtx->dr[7];
10423	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10424	&& X86_DR7_ANY_RW_IO(uDr7)
10425	&& (pMixedCtx->cr4 & X86_CR4_DE))
10426	\|\| DBGFBpIsHwIoArmed(pVM)))
10427	{
10428	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10429
10430	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10431	VMMRZCallRing3Disable(pVCpu);
10432	HM_DISABLE_PREEMPT_IF_NEEDED();
10433
10434	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /fDr6/);
10435
10436	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10437	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10438	{
10439	/* Raise #DB. */
10440	if (fIsGuestDbgActive)
10441	ASMSetDR6(pMixedCtx->dr[6]);
10442	if (pMixedCtx->dr[7] != uDr7)
10443	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10444
10445	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10446	}
10447	/* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10448	else if ( rcStrict2 != VINF_SUCCESS
10449	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
10450	rcStrict = rcStrict2;
10451
10452	HM_RESTORE_PREEMPT_IF_NEEDED();
10453	VMMRZCallRing3Enable(pVCpu);
10454	}
10455	}
10456
10457	#ifdef DEBUG
10458	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10459	Assert(!fIOWrite);
10460	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10461	Assert(fIOWrite);
10462	else
10463	{
10464	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10465	* statuses, that the VMM device and some others may return. See
10466	* IOM_SUCCESS() for guidance. */
10467	AssertMsg( RT_FAILURE(rcStrict)
10468	\|\| rcStrict == VINF_SUCCESS
10469	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
10470	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
10471	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
10472	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10473	}
10474	#endif
10475
10476	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10477	return VBOXSTRICTRC_TODO(rcStrict);
10478	}
10479
10480
10481	/**
10482	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10483	* VM-exit.
10484	*/
10485	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10486	{
10487	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10488
10489	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
10490	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10491	AssertRCReturn(rc, rc);
10492	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10493	{
10494	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10495	AssertRCReturn(rc, rc);
10496	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10497	{
10498	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10499
10500	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
10501	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
10502	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
10503	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
10504	{
10505	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10506	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10507
10508	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10509	Assert(!pVCpu->hm.s.Event.fPending);
10510	pVCpu->hm.s.Event.fPending = true;
10511	pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10512	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10513	AssertRCReturn(rc, rc);
10514	if (fErrorCodeValid)
10515	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10516	else
10517	pVCpu->hm.s.Event.u32ErrCode = 0;
10518	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10519	&& uVector == X86_XCPT_PF)
10520	{
10521	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10522	}
10523
10524	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10525	}
10526	}
10527	}
10528
10529	/** @todo Emulate task switch someday, currently just going back to ring-3 for
10530	* emulation. */
10531	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10532	return VERR_EM_INTERPRETER;
10533	}
10534
10535
10536	/**
10537	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10538	*/
10539	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10540	{
10541	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10542	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10543	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10544	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10545	AssertRCReturn(rc, rc);
10546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10547	return VINF_EM_DBG_STEPPED;
10548	}
10549
10550
10551	/**
10552	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10553	*/
10554	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10555	{
10556	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10557
10558	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10559	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10560	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10561	return VINF_SUCCESS;
10562	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10563	return rc;
10564
10565	#if 0
10566	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10567	* just sync the whole thing. */
10568	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10569	#else
10570	/* Aggressive state sync. for now. */
10571	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10572	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10573	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10574	#endif
10575	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10576	AssertRCReturn(rc, rc);
10577
10578	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10579	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10580	switch (uAccessType)
10581	{
10582	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10583	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10584	{
10585	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10586	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10587	{
10588	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10589	}
10590
10591	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10592	GCPhys &= PAGE_BASE_GC_MASK;
10593	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10594	PVM pVM = pVCpu->CTX_SUFF(pVM);
10595	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10596	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10597
10598	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10599	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10600	CPUMCTX2CORE(pMixedCtx), GCPhys);
10601	rc = VBOXSTRICTRC_VAL(rc2);
10602	Log4(("ApicAccess rc=%d\n", rc));
10603	if ( rc == VINF_SUCCESS
10604	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10605	\|\| rc == VERR_PAGE_NOT_PRESENT)
10606	{
10607	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10608	\| HM_CHANGED_GUEST_RSP
10609	\| HM_CHANGED_GUEST_RFLAGS
10610	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10611	rc = VINF_SUCCESS;
10612	}
10613	break;
10614	}
10615
10616	default:
10617	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10618	rc = VINF_EM_RAW_EMULATE_INSTR;
10619	break;
10620	}
10621
10622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10623	return rc;
10624	}
10625
10626
10627	/**
10628	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10629	* VM-exit.
10630	*/
10631	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10632	{
10633	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10634
10635	/* We should -not- get this VM-exit if the guest's debug registers were active. */
10636	if (pVmxTransient->fWasGuestDebugStateActive)
10637	{
10638	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10639	HMVMX_RETURN_UNEXPECTED_EXIT();
10640	}
10641
10642	int rc = VERR_INTERNAL_ERROR_5;
10643	if ( !DBGFIsStepping(pVCpu)
10644	&& !pVCpu->hm.s.fSingleInstruction
10645	&& !pVmxTransient->fWasHyperDebugStateActive)
10646	{
10647	/* Don't intercept MOV DRx and #DB any more. */
10648	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10649	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10650	AssertRCReturn(rc, rc);
10651
10652	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10653	{
10654	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10655	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10656	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10657	AssertRCReturn(rc, rc);
10658	#endif
10659	}
10660
10661	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10662	VMMRZCallRing3Disable(pVCpu);
10663	HM_DISABLE_PREEMPT_IF_NEEDED();
10664
10665	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10666	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10667	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
10668
10669	HM_RESTORE_PREEMPT_IF_NEEDED();
10670	VMMRZCallRing3Enable(pVCpu);
10671
10672	#ifdef VBOX_WITH_STATISTICS
10673	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10674	AssertRCReturn(rc, rc);
10675	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10676	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10677	else
10678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10679	#endif
10680	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10681	return VINF_SUCCESS;
10682	}
10683
10684	/*
10685	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
10686	* Update the segment registers and DR7 from the CPU.
10687	*/
10688	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10689	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10690	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10691	AssertRCReturn(rc, rc);
10692	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10693
10694	PVM pVM = pVCpu->CTX_SUFF(pVM);
10695	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10696	{
10697	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10698	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10699	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10700	if (RT_SUCCESS(rc))
10701	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10703	}
10704	else
10705	{
10706	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10707	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10708	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10709	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10710	}
10711
10712	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
10713	if (RT_SUCCESS(rc))
10714	{
10715	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10716	AssertRCReturn(rc2, rc2);
10717	}
10718	return rc;
10719	}
10720
10721
10722	/**
10723	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10724	* Conditional VM-exit.
10725	*/
10726	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10727	{
10728	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10729	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10730
10731	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10732	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10733	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10734	return VINF_SUCCESS;
10735	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10736	return rc;
10737
10738	RTGCPHYS GCPhys = 0;
10739	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10740
10741	#if 0
10742	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10743	#else
10744	/* Aggressive state sync. for now. */
10745	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10746	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10747	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10748	#endif
10749	AssertRCReturn(rc, rc);
10750
10751	/*
10752	* If we succeed, resume guest execution.
10753	* If we fail in interpreting the instruction because we couldn't get the guest physical address
10754	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10755	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10756	* weird case. See @bugref{6043}.
10757	*/
10758	PVM pVM = pVCpu->CTX_SUFF(pVM);
10759	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10760	rc = VBOXSTRICTRC_VAL(rc2);
10761	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10762	if ( rc == VINF_SUCCESS
10763	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10764	\|\| rc == VERR_PAGE_NOT_PRESENT)
10765	{
10766	/* Successfully handled MMIO operation. */
10767	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10768	\| HM_CHANGED_GUEST_RSP
10769	\| HM_CHANGED_GUEST_RFLAGS
10770	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
10771	rc = VINF_SUCCESS;
10772	}
10773	return rc;
10774	}
10775
10776
10777	/**
10778	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10779	* VM-exit.
10780	*/
10781	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10782	{
10783	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10784	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10785
10786	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10787	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10788	if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10789	return VINF_SUCCESS;
10790	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10791	return rc;
10792
10793	RTGCPHYS GCPhys = 0;
10794	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10795	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10796	#if 0
10797	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10798	#else
10799	/* Aggressive state sync. for now. */
10800	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10801	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10802	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10803	#endif
10804	AssertRCReturn(rc, rc);
10805
10806	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10807	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10808
10809	RTGCUINT uErrorCode = 0;
10810	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10811	uErrorCode \|= X86_TRAP_PF_ID;
10812	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10813	uErrorCode \|= X86_TRAP_PF_RW;
10814	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10815	uErrorCode \|= X86_TRAP_PF_P;
10816
10817	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10818
10819	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10820	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10821
10822	/* Handle the pagefault trap for the nested shadow table. */
10823	PVM pVM = pVCpu->CTX_SUFF(pVM);
10824	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10825	TRPMResetTrap(pVCpu);
10826
10827	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10828	if ( rc == VINF_SUCCESS
10829	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
10830	\|\| rc == VERR_PAGE_NOT_PRESENT)
10831	{
10832	/* Successfully synced our nested page tables. */
10833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10834	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10835	\| HM_CHANGED_GUEST_RSP
10836	\| HM_CHANGED_GUEST_RFLAGS);
10837	return VINF_SUCCESS;
10838	}
10839
10840	Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
10841	return rc;
10842	}
10843
10844	/** @} */
10845
10846	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10847	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10848	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10849
10850	/** @name VM-exit exception handlers.
10851	* @{
10852	*/
10853
10854	/**
10855	* VM-exit exception handler for #MF (Math Fault: floating point exception).
10856	*/
10857	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10858	{
10859	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10861
10862	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10863	AssertRCReturn(rc, rc);
10864
10865	if (!(pMixedCtx->cr0 & X86_CR0_NE))
10866	{
10867	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
10868	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
10869	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10870	AssertRCReturn(rc2, rc2);
10871	return rc;
10872	}
10873
10874	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10875	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10876	return rc;
10877	}
10878
10879
10880	/**
10881	* VM-exit exception handler for #BP (Breakpoint exception).
10882	*/
10883	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10884	{
10885	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10886	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10887
10888	/** @todo Try optimize this by not saving the entire guest state unless
10889	* really needed. */
10890	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10891	AssertRCReturn(rc, rc);
10892
10893	PVM pVM = pVCpu->CTX_SUFF(pVM);
10894	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10895	if (rc == VINF_EM_RAW_GUEST_TRAP)
10896	{
10897	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10898	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10899	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10900	AssertRCReturn(rc, rc);
10901
10902	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10903	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10904	}
10905
10906	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
10907	return rc;
10908	}
10909
10910
10911	/**
10912	* VM-exit exception handler for #DB (Debug exception).
10913	*/
10914	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10915	{
10916	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10918	Log6(("XcptDB\n"));
10919
10920	/*
10921	* Get the DR6-like values from the exit qualification and pass it to DBGF
10922	* for processing.
10923	*/
10924	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10925	AssertRCReturn(rc, rc);
10926
10927	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10928	uint64_t uDR6 = X86_DR6_INIT_VAL;
10929	uDR6 \|= ( pVmxTransient->uExitQualification
10930	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
10931
10932	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10933	if (rc == VINF_EM_RAW_GUEST_TRAP)
10934	{
10935	/*
10936	* The exception was for the guest. Update DR6, DR7.GD and
10937	* IA32_DEBUGCTL.LBR before forwarding it.
10938	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10939	*/
10940	VMMRZCallRing3Disable(pVCpu);
10941	HM_DISABLE_PREEMPT_IF_NEEDED();
10942
10943	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10944	pMixedCtx->dr[6] \|= uDR6;
10945	if (CPUMIsGuestDebugStateActive(pVCpu))
10946	ASMSetDR6(pMixedCtx->dr[6]);
10947
10948	HM_RESTORE_PREEMPT_IF_NEEDED();
10949	VMMRZCallRing3Enable(pVCpu);
10950
10951	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10952	AssertRCReturn(rc, rc);
10953
10954	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10955	pMixedCtx->dr[7] &= ~X86_DR7_GD;
10956
10957	/* Paranoia. */
10958	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10959	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
10960
10961	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10962	AssertRCReturn(rc, rc);
10963
10964	/*
10965	* Raise #DB in the guest.
10966	*
10967	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
10968	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
10969	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
10970	*
10971	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
10972	*/
10973	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10974	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10975	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10976	AssertRCReturn(rc, rc);
10977	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
10978	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
10979	return VINF_SUCCESS;
10980	}
10981
10982	/*
10983	* Not a guest trap, must be a hypervisor related debug event then.
10984	* Update DR6 in case someone is interested in it.
10985	*/
10986	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10987	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10988	CPUMSetHyperDR6(pVCpu, uDR6);
10989
10990	return rc;
10991	}
10992
10993
10994	/**
10995	* VM-exit exception handler for #NM (Device-not-available exception: floating
10996	* point exception).
10997	*/
10998	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10999	{
11000	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11001
11002	/* We require CR0 and EFER. EFER is always up-to-date. */
11003	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11004	AssertRCReturn(rc, rc);
11005
11006	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11007	VMMRZCallRing3Disable(pVCpu);
11008	HM_DISABLE_PREEMPT_IF_NEEDED();
11009
11010	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11011	if (pVmxTransient->fWasGuestFPUStateActive)
11012	{
11013	rc = VINF_EM_RAW_GUEST_TRAP;
11014	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11015	}
11016	else
11017	{
11018	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11019	Assert(!pVmxTransient->fWasGuestFPUStateActive);
11020	#endif
11021	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11022	Assert(rc == VINF_EM_RAW_GUEST_TRAP \|\| (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11023	}
11024
11025	HM_RESTORE_PREEMPT_IF_NEEDED();
11026	VMMRZCallRing3Enable(pVCpu);
11027
11028	if (rc == VINF_SUCCESS)
11029	{
11030	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11031	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11032	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11033	pVCpu->hm.s.fUseGuestFpu = true;
11034	}
11035	else
11036	{
11037	/* Forward #NM to the guest. */
11038	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11039	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11040	AssertRCReturn(rc, rc);
11041	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11042	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
11043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11044	}
11045
11046	return VINF_SUCCESS;
11047	}
11048
11049
11050	/**
11051	* VM-exit exception handler for #GP (General-protection exception).
11052	*
11053	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11054	*/
11055	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11056	{
11057	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11059
11060	int rc = VERR_INTERNAL_ERROR_5;
11061	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11062	{
11063	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11064	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11065	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11066	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11067	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11068	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11069	AssertRCReturn(rc, rc);
11070	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntErrorCode,
11071	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
11072	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11073	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11074	return rc;
11075	#else
11076	/* We don't intercept #GP. */
11077	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11078	NOREF(pVmxTransient);
11079	return VERR_VMX_UNEXPECTED_EXCEPTION;
11080	#endif
11081	}
11082
11083	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11084	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11085
11086	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11087	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11088	AssertRCReturn(rc, rc);
11089
11090	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11091	uint32_t cbOp = 0;
11092	PVM pVM = pVCpu->CTX_SUFF(pVM);
11093	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11094	if (RT_SUCCESS(rc))
11095	{
11096	rc = VINF_SUCCESS;
11097	Assert(cbOp == pDis->cbInstr);
11098	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11099	switch (pDis->pCurInstr->uOpcode)
11100	{
11101	case OP_CLI:
11102	{
11103	pMixedCtx->eflags.Bits.u1IF = 0;
11104	pMixedCtx->rip += pDis->cbInstr;
11105	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11107	break;
11108	}
11109
11110	case OP_STI:
11111	{
11112	pMixedCtx->eflags.Bits.u1IF = 1;
11113	pMixedCtx->rip += pDis->cbInstr;
11114	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11115	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11116	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
11117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11118	break;
11119	}
11120
11121	case OP_HLT:
11122	{
11123	rc = VINF_EM_HALT;
11124	pMixedCtx->rip += pDis->cbInstr;
11125	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11126	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11127	break;
11128	}
11129
11130	case OP_POPF:
11131	{
11132	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11133	uint32_t cbParm = 0;
11134	uint32_t uMask = 0;
11135	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11136	{
11137	cbParm = 4;
11138	uMask = 0xffffffff;
11139	}
11140	else
11141	{
11142	cbParm = 2;
11143	uMask = 0xffff;
11144	}
11145
11146	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
11147	RTGCPTR GCPtrStack = 0;
11148	X86EFLAGS Eflags;
11149	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11150	&GCPtrStack);
11151	if (RT_SUCCESS(rc))
11152	{
11153	Assert(sizeof(Eflags.u32) >= cbParm);
11154	Eflags.u32 = 0;
11155	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11156	}
11157	if (RT_FAILURE(rc))
11158	{
11159	rc = VERR_EM_INTERPRETER;
11160	break;
11161	}
11162	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11163	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11164	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11165	pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11166	pMixedCtx->esp += cbParm;
11167	pMixedCtx->esp &= uMask;
11168	pMixedCtx->rip += pDis->cbInstr;
11169
11170	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11171	\| HM_CHANGED_GUEST_RSP
11172	\| HM_CHANGED_GUEST_RFLAGS);
11173	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11174	break;
11175	}
11176
11177	case OP_PUSHF:
11178	{
11179	uint32_t cbParm = 0;
11180	uint32_t uMask = 0;
11181	if (pDis->fPrefix & DISPREFIX_OPSIZE)
11182	{
11183	cbParm = 4;
11184	uMask = 0xffffffff;
11185	}
11186	else
11187	{
11188	cbParm = 2;
11189	uMask = 0xffff;
11190	}
11191
11192	/* Get the stack pointer & push the contents of eflags onto the stack. */
11193	RTGCPTR GCPtrStack = 0;
11194	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11195	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11196	if (RT_FAILURE(rc))
11197	{
11198	rc = VERR_EM_INTERPRETER;
11199	break;
11200	}
11201	X86EFLAGS Eflags = pMixedCtx->eflags;
11202	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11203	Eflags.Bits.u1RF = 0;
11204	Eflags.Bits.u1VM = 0;
11205
11206	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11207	if (RT_FAILURE(rc))
11208	{
11209	rc = VERR_EM_INTERPRETER;
11210	break;
11211	}
11212	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11213	pMixedCtx->esp -= cbParm;
11214	pMixedCtx->esp &= uMask;
11215	pMixedCtx->rip += pDis->cbInstr;
11216	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP);
11217	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11218	break;
11219	}
11220
11221	case OP_IRET:
11222	{
11223	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11224	* instruction reference. */
11225	RTGCPTR GCPtrStack = 0;
11226	uint32_t uMask = 0xffff;
11227	uint16_t aIretFrame[3];
11228	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
11229	{
11230	rc = VERR_EM_INTERPRETER;
11231	break;
11232	}
11233	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11234	&GCPtrStack);
11235	if (RT_SUCCESS(rc))
11236	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11237	if (RT_FAILURE(rc))
11238	{
11239	rc = VERR_EM_INTERPRETER;
11240	break;
11241	}
11242	pMixedCtx->eip = 0;
11243	pMixedCtx->ip = aIretFrame[0];
11244	pMixedCtx->cs.Sel = aIretFrame[1];
11245	pMixedCtx->cs.ValidSel = aIretFrame[1];
11246	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11247	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11248	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11249	pMixedCtx->sp += sizeof(aIretFrame);
11250	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11251	\| HM_CHANGED_GUEST_SEGMENT_REGS
11252	\| HM_CHANGED_GUEST_RSP
11253	\| HM_CHANGED_GUEST_RFLAGS);
11254	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11256	break;
11257	}
11258
11259	case OP_INT:
11260	{
11261	uint16_t uVector = pDis->Param1.uValue & 0xff;
11262	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11263	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11264	break;
11265	}
11266
11267	case OP_INTO:
11268	{
11269	if (pMixedCtx->eflags.Bits.u1OF)
11270	{
11271	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11273	}
11274	break;
11275	}
11276
11277	default:
11278	{
11279	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11280	EMCODETYPE_SUPERVISOR);
11281	rc = VBOXSTRICTRC_VAL(rc2);
11282	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11283	Log4(("#GP rc=%Rrc\n", rc));
11284	break;
11285	}
11286	}
11287	}
11288	else
11289	rc = VERR_EM_INTERPRETER;
11290
11291	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
11292	("#GP Unexpected rc=%Rrc\n", rc));
11293	return rc;
11294	}
11295
11296
11297	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11298	/**
11299	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11300	* the exception reported in the VMX transient structure back into the VM.
11301	*
11302	* @remarks Requires uExitIntInfo in the VMX transient structure to be
11303	* up-to-date.
11304	*/
11305	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11306	{
11307	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11308
11309	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11310	hmR0VmxCheckExitDueToEventDelivery(). */
11311	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11312	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11313	AssertRCReturn(rc, rc);
11314	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11315
11316	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11317	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11318	return VINF_SUCCESS;
11319	}
11320	#endif
11321
11322
11323	/**
11324	* VM-exit exception handler for #PF (Page-fault exception).
11325	*/
11326	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11327	{
11328	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11329	PVM pVM = pVCpu->CTX_SUFF(pVM);
11330	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11331	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11332	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11333	AssertRCReturn(rc, rc);
11334
11335	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
11336	if (pVM->hm.s.fNestedPaging)
11337	{
11338	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11339	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11340	{
11341	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11342	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11343	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11344	}
11345	else
11346	{
11347	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11348	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11349	Log4(("Pending #DF due to vectoring #PF. NP\n"));
11350	}
11351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11352	return rc;
11353	}
11354	#else
11355	Assert(!pVM->hm.s.fNestedPaging);
11356	NOREF(pVM);
11357	#endif
11358
11359	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11360	AssertRCReturn(rc, rc);
11361
11362	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11363	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11364
11365	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11366	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11367	(RTGCPTR)pVmxTransient->uExitQualification);
11368
11369	Log4(("#PF: rc=%Rrc\n", rc));
11370	if (rc == VINF_SUCCESS)
11371	{
11372	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11373	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
11374	* memory? We don't update the whole state here... */
11375	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11376	\| HM_CHANGED_GUEST_RSP
11377	\| HM_CHANGED_GUEST_RFLAGS
11378	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
11379	TRPMResetTrap(pVCpu);
11380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11381	return rc;
11382	}
11383	else if (rc == VINF_EM_RAW_GUEST_TRAP)
11384	{
11385	if (!pVmxTransient->fVectoringPF)
11386	{
11387	/* It's a guest page fault and needs to be reflected to the guest. */
11388	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11389	TRPMResetTrap(pVCpu);
11390	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11391	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11392	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11393	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11394	}
11395	else
11396	{
11397	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11398	TRPMResetTrap(pVCpu);
11399	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11400	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11401	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11402	}
11403
11404	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11405	return VINF_SUCCESS;
11406	}
11407
11408	TRPMResetTrap(pVCpu);
11409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11410	return rc;
11411	}
11412
11413	/** @} */
11414

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 49864

Download in other formats: