HMSVMR0.cpp@ 70303

Last change on this file since 70303 was 70303, checked in by vboxsync, 7 years ago
VMM/HMSVMR0: Fix debug assertion while stepping through the VM debugger.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 298.2 KB

Line
1	/* $Id: HMSVMR0.cpp 70303 2017-12-22 08:01:30Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/gim.h>
33	#include <VBox/vmm/apic.h>
34	#include "HMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include "HMSVMR0.h"
37	#include "dtrace/VBoxVMM.h"
38
39	#define HMSVM_USE_IEM_EVENT_REFLECTION
40	#ifdef DEBUG_ramshankar
41	# define HMSVM_SYNC_FULL_GUEST_STATE
42	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
43	# define HMSVM_ALWAYS_TRAP_PF
44	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
45	#endif
46
47
48	/*********************************************************************************************************************************
49	* Defined Constants And Macros *
50	*********************************************************************************************************************************/
51	#ifdef VBOX_WITH_STATISTICS
52	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
53	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
54	if ((u64ExitCode) == SVM_EXIT_NPF) \
55	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
56	else \
57	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
58	} while (0)
59	#else
60	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
61	#endif
62
63	/** If we decide to use a function table approach this can be useful to
64	* switch to a "static DECLCALLBACK(int)". */
65	#define HMSVM_EXIT_DECL static int
66
67	/** Macro for checking and returning from the using function for
68	* \#VMEXIT intercepts that maybe caused during delivering of another
69	* event in the guest. */
70	#define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
71	do \
72	{ \
73	int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
74	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* likely */ } \
75	else if (rc == VINF_HM_DOUBLE_FAULT) \
76	return VINF_SUCCESS; \
77	else \
78	return rc; \
79	} while (0)
80
81	/**
82	* Updates interrupt shadow for the current RIP.
83	*/
84	#define HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx) \
85	do { \
86	/* Update interrupt shadow. */ \
87	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS) \
88	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu)) \
89	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); \
90	} while (0)
91
92	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
93	* instruction that exited. */
94	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
95	do { \
96	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
97	(a_rc) = VINF_EM_DBG_STEPPED; \
98	} while (0)
99
100	/** Assert that preemption is disabled or covered by thread-context hooks. */
101	#define HMSVM_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
102	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
103
104	/** Assert that we haven't migrated CPUs when thread-context hooks are not
105	* used. */
106	#define HMSVM_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
107	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
108	("Illegal migration! Entered on CPU %u Current %u\n", \
109	pVCpu->hm.s.idEnteredCpu, RTMpCpuId()));
110
111	/** Assert that we're not executing a nested-guest. */
112	#ifdef VBOX_WITH_NESTED_HWVIRT
113	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
114	#else
115	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
116	#endif
117
118	/** Assert that we're executing a nested-guest. */
119	#ifdef VBOX_WITH_NESTED_HWVIRT
120	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
121	#else
122	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
123	#endif
124
125	/**
126	* Exception bitmap mask for all contributory exceptions.
127	*
128	* Page fault is deliberately excluded here as it's conditional as to whether
129	* it's contributory or benign. Page faults are handled separately.
130	*/
131	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
132	\| RT_BIT(X86_XCPT_DE))
133
134	/**
135	* Mandatory/unconditional guest control intercepts.
136	*
137	* SMIs can and do happen in normal operation. We need not intercept them
138	* while executing the guest or nested-guest.
139	*/
140	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
141	\| SVM_CTRL_INTERCEPT_NMI \
142	\| SVM_CTRL_INTERCEPT_INIT \
143	\| SVM_CTRL_INTERCEPT_RDPMC \
144	\| SVM_CTRL_INTERCEPT_CPUID \
145	\| SVM_CTRL_INTERCEPT_RSM \
146	\| SVM_CTRL_INTERCEPT_HLT \
147	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
148	\| SVM_CTRL_INTERCEPT_MSR_PROT \
149	\| SVM_CTRL_INTERCEPT_INVLPGA \
150	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
151	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
152	\| SVM_CTRL_INTERCEPT_VMRUN \
153	\| SVM_CTRL_INTERCEPT_VMMCALL \
154	\| SVM_CTRL_INTERCEPT_VMLOAD \
155	\| SVM_CTRL_INTERCEPT_VMSAVE \
156	\| SVM_CTRL_INTERCEPT_STGI \
157	\| SVM_CTRL_INTERCEPT_CLGI \
158	\| SVM_CTRL_INTERCEPT_SKINIT \
159	\| SVM_CTRL_INTERCEPT_WBINVD \
160	\| SVM_CTRL_INTERCEPT_MONITOR \
161	\| SVM_CTRL_INTERCEPT_MWAIT \
162	\| SVM_CTRL_INTERCEPT_XSETBV)
163
164	/** @name VMCB Clean Bits.
165	*
166	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
167	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
168	* memory.
169	*
170	* @{ */
171	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
172	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
173	/** I/O permission bitmap, MSR permission bitmap. */
174	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
175	/** ASID. */
176	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
177	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
178	V_INTR_VECTOR. */
179	#define HMSVM_VMCB_CLEAN_TPR RT_BIT(3)
180	/** Nested Paging: Nested CR3 (nCR3), PAT. */
181	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
182	/** Control registers (CR0, CR3, CR4, EFER). */
183	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
184	/** Debug registers (DR6, DR7). */
185	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
186	/** GDT, IDT limit and base. */
187	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
188	/** Segment register: CS, SS, DS, ES limit and base. */
189	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
190	/** CR2.*/
191	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
192	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
193	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
194	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
195	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
196	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
197	/** Mask of all valid VMCB Clean bits. */
198	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
199	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
200	\| HMSVM_VMCB_CLEAN_ASID \
201	\| HMSVM_VMCB_CLEAN_TPR \
202	\| HMSVM_VMCB_CLEAN_NP \
203	\| HMSVM_VMCB_CLEAN_CRX_EFER \
204	\| HMSVM_VMCB_CLEAN_DRX \
205	\| HMSVM_VMCB_CLEAN_DT \
206	\| HMSVM_VMCB_CLEAN_SEG \
207	\| HMSVM_VMCB_CLEAN_CR2 \
208	\| HMSVM_VMCB_CLEAN_LBR \
209	\| HMSVM_VMCB_CLEAN_AVIC)
210	/** @} */
211
212	/** @name SVM transient.
213	*
214	* A state structure for holding miscellaneous information across AMD-V
215	* VMRUN/\#VMEXIT operation, restored after the transition.
216	*
217	* @{ */
218	typedef struct SVMTRANSIENT
219	{
220	/** The host's rflags/eflags. */
221	RTCCUINTREG fEFlags;
222	#if HC_ARCH_BITS == 32
223	uint32_t u32Alignment0;
224	#endif
225
226	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
227	uint64_t u64ExitCode;
228	/** The guest's TPR value used for TPR shadowing. */
229	uint8_t u8GuestTpr;
230	/** Alignment. */
231	uint8_t abAlignment0[7];
232
233	/** Whether the guest FPU state was active at the time of \#VMEXIT. */
234	bool fWasGuestFPUStateActive;
235	/** Whether the guest debug state was active at the time of \#VMEXIT. */
236	bool fWasGuestDebugStateActive;
237	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
238	bool fWasHyperDebugStateActive;
239	/** Whether the TSC offset mode needs to be updated. */
240	bool fUpdateTscOffsetting;
241	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
242	bool fRestoreTscAuxMsr;
243	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
244	* contributary exception or a page-fault. */
245	bool fVectoringDoublePF;
246	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
247	* external interrupt or NMI. */
248	bool fVectoringPF;
249	} SVMTRANSIENT, *PSVMTRANSIENT;
250	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
251	AssertCompileMemberAlignment(SVMTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
252	/** @} */
253
254	/**
255	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
256	*/
257	typedef enum SVMMSREXITREAD
258	{
259	/** Reading this MSR causes a \#VMEXIT. */
260	SVMMSREXIT_INTERCEPT_READ = 0xb,
261	/** Reading this MSR does not cause a \#VMEXIT. */
262	SVMMSREXIT_PASSTHRU_READ
263	} SVMMSREXITREAD;
264
265	/**
266	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
267	*/
268	typedef enum SVMMSREXITWRITE
269	{
270	/** Writing to this MSR causes a \#VMEXIT. */
271	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
272	/** Writing to this MSR does not cause a \#VMEXIT. */
273	SVMMSREXIT_PASSTHRU_WRITE
274	} SVMMSREXITWRITE;
275
276	/**
277	* SVM \#VMEXIT handler.
278	*
279	* @returns VBox status code.
280	* @param pVCpu The cross context virtual CPU structure.
281	* @param pMixedCtx Pointer to the guest-CPU context.
282	* @param pSvmTransient Pointer to the SVM-transient structure.
283	*/
284	typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
285
286
287	/*********************************************************************************************************************************
288	* Internal Functions *
289	*********************************************************************************************************************************/
290	static void hmR0SvmSetMsrPermission(PSVMVMCB pVmcb, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
291	SVMMSREXITWRITE enmWrite);
292	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
293	static void hmR0SvmLeave(PVMCPU pVCpu);
294
295	/** @name \#VMEXIT handlers.
296	* @{
297	*/
298	static FNSVMEXITHANDLER hmR0SvmExitIntr;
299	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
300	static FNSVMEXITHANDLER hmR0SvmExitInvd;
301	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
302	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
303	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
304	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
305	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
306	static FNSVMEXITHANDLER hmR0SvmExitHlt;
307	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
308	static FNSVMEXITHANDLER hmR0SvmExitMwait;
309	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
310	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
311	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
312	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
313	static FNSVMEXITHANDLER hmR0SvmExitMsr;
314	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
315	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
316	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
317	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
318	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
319	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
320	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
321	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
322	static FNSVMEXITHANDLER hmR0SvmExitPause;
323	static FNSVMEXITHANDLER hmR0SvmExitIret;
324	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
325	static FNSVMEXITHANDLER hmR0SvmExitXcptNM;
326	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
327	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
328	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
329	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
330	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
331	#ifdef VBOX_WITH_NESTED_HWVIRT
332	static FNSVMEXITHANDLER hmR0SvmExitXcptPFNested;
333	static FNSVMEXITHANDLER hmR0SvmExitClgi;
334	static FNSVMEXITHANDLER hmR0SvmExitStgi;
335	static FNSVMEXITHANDLER hmR0SvmExitVmload;
336	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
337	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
338	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
339	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
340	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
341	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
342	#endif
343	/** @} */
344
345	static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient);
346	#ifdef VBOX_WITH_NESTED_HWVIRT
347	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
348	#endif
349
350
351	/*********************************************************************************************************************************
352	* Global Variables *
353	*********************************************************************************************************************************/
354	/** Ring-0 memory object for the IO bitmap. */
355	RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
356	/** Physical address of the IO bitmap. */
357	RTHCPHYS g_HCPhysIOBitmap = 0;
358	/** Pointer to the IO bitmap. */
359	R0PTRTYPE(void *) g_pvIOBitmap = NULL;
360
361	#ifdef VBOX_WITH_NESTED_HWVIRT
362	/** Ring-0 memory object for the nested-guest MSRPM bitmap. */
363	RTR0MEMOBJ g_hMemObjNstGstMsrBitmap = NIL_RTR0MEMOBJ;
364	/** Physical address of the nested-guest MSRPM bitmap. */
365	RTHCPHYS g_HCPhysNstGstMsrBitmap = 0;
366	/** Pointer to the nested-guest MSRPM bitmap. */
367	R0PTRTYPE(void *) g_pvNstGstMsrBitmap = NULL;
368	#endif
369
370	/**
371	* Sets up and activates AMD-V on the current CPU.
372	*
373	* @returns VBox status code.
374	* @param pCpu Pointer to the CPU info struct.
375	* @param pVM The cross context VM structure. Can be
376	* NULL after a resume!
377	* @param pvCpuPage Pointer to the global CPU page.
378	* @param HCPhysCpuPage Physical address of the global CPU page.
379	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
380	* @param pvArg Unused on AMD-V.
381	*/
382	VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
383	void *pvArg)
384	{
385	Assert(!fEnabledByHost);
386	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
387	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
388	Assert(pvCpuPage); NOREF(pvCpuPage);
389	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
390
391	NOREF(pvArg);
392	NOREF(fEnabledByHost);
393
394	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
395	RTCCUINTREG fEFlags = ASMIntDisableFlags();
396
397	/*
398	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
399	*/
400	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
401	if (u64HostEfer & MSR_K6_EFER_SVME)
402	{
403	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
404	if ( pVM
405	&& pVM->hm.s.svm.fIgnoreInUseError)
406	{
407	pCpu->fIgnoreAMDVInUseError = true;
408	}
409
410	if (!pCpu->fIgnoreAMDVInUseError)
411	{
412	ASMSetFlags(fEFlags);
413	return VERR_SVM_IN_USE;
414	}
415	}
416
417	/* Turn on AMD-V in the EFER MSR. */
418	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
419
420	/* Write the physical page address where the CPU will store the host state while executing the VM. */
421	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
422
423	/* Restore interrupts. */
424	ASMSetFlags(fEFlags);
425
426	/*
427	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all non-zero ASIDs
428	* when enabling SVM. AMD doesn't have an SVM instruction to flush all ASIDs (flushing is done
429	* upon VMRUN). Therefore, flag that we need to flush the TLB entirely with before executing any
430	* guest code.
431	*/
432	pCpu->fFlushAsidBeforeUse = true;
433
434	/*
435	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
436	*/
437	++pCpu->cTlbFlushes;
438
439	return VINF_SUCCESS;
440	}
441
442
443	/**
444	* Deactivates AMD-V on the current CPU.
445	*
446	* @returns VBox status code.
447	* @param pCpu Pointer to the CPU info struct.
448	* @param pvCpuPage Pointer to the global CPU page.
449	* @param HCPhysCpuPage Physical address of the global CPU page.
450	*/
451	VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
452	{
453	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
454	AssertReturn( HCPhysCpuPage
455	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
456	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
457	NOREF(pCpu);
458
459	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
460	RTCCUINTREG fEFlags = ASMIntDisableFlags();
461
462	/* Turn off AMD-V in the EFER MSR. */
463	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
464	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
465
466	/* Invalidate host state physical address. */
467	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
468
469	/* Restore interrupts. */
470	ASMSetFlags(fEFlags);
471
472	return VINF_SUCCESS;
473	}
474
475
476	/**
477	* Does global AMD-V initialization (called during module initialization).
478	*
479	* @returns VBox status code.
480	*/
481	VMMR0DECL(int) SVMR0GlobalInit(void)
482	{
483	/*
484	* Allocate 12 KB for the IO bitmap. Since this is non-optional and we always intercept all IO accesses, it's done
485	* once globally here instead of per-VM.
486	*/
487	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
488	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
489	if (RT_FAILURE(rc))
490	return rc;
491
492	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
493	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
494
495	/* Set all bits to intercept all IO accesses. */
496	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
497
498	#ifdef VBOX_WITH_NESTED_HWVIRT
499	/*
500	* Allocate 8 KB for the MSR permission bitmap for the nested-guest.
501	*/
502	Assert(g_hMemObjNstGstMsrBitmap == NIL_RTR0MEMOBJ);
503	rc = RTR0MemObjAllocCont(&g_hMemObjNstGstMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
504	if (RT_FAILURE(rc))
505	return rc;
506
507	g_pvNstGstMsrBitmap = RTR0MemObjAddress(g_hMemObjNstGstMsrBitmap);
508	g_HCPhysNstGstMsrBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjNstGstMsrBitmap, 0 /* iPage */);
509
510	/* Set all bits to intercept all MSR accesses. */
511	ASMMemFill32(g_pvNstGstMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
512	#endif
513
514	return VINF_SUCCESS;
515	}
516
517
518	/**
519	* Does global AMD-V termination (called during module termination).
520	*/
521	VMMR0DECL(void) SVMR0GlobalTerm(void)
522	{
523	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
524	{
525	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
526	g_pvIOBitmap = NULL;
527	g_HCPhysIOBitmap = 0;
528	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
529	}
530
531	#ifdef VBOX_WITH_NESTED_HWVIRT
532	if (g_hMemObjNstGstMsrBitmap != NIL_RTR0MEMOBJ)
533	{
534	RTR0MemObjFree(g_hMemObjNstGstMsrBitmap, true /* fFreeMappings */);
535	g_pvNstGstMsrBitmap = NULL;
536	g_HCPhysNstGstMsrBitmap = 0;
537	g_hMemObjNstGstMsrBitmap = NIL_RTR0MEMOBJ;
538	}
539	#endif
540	}
541
542
543	/**
544	* Frees any allocated per-VCPU structures for a VM.
545	*
546	* @param pVM The cross context VM structure.
547	*/
548	DECLINLINE(void) hmR0SvmFreeStructs(PVM pVM)
549	{
550	for (uint32_t i = 0; i < pVM->cCpus; i++)
551	{
552	PVMCPU pVCpu = &pVM->aCpus[i];
553	AssertPtr(pVCpu);
554
555	if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
556	{
557	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
558	pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
559	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
560	}
561
562	if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
563	{
564	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
565	pVCpu->hm.s.svm.pVmcb = NULL;
566	pVCpu->hm.s.svm.HCPhysVmcb = 0;
567	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
568	}
569
570	if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
571	{
572	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
573	pVCpu->hm.s.svm.pvMsrBitmap = NULL;
574	pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
575	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
576	}
577	}
578	}
579
580
581	/**
582	* Does per-VM AMD-V initialization.
583	*
584	* @returns VBox status code.
585	* @param pVM The cross context VM structure.
586	*/
587	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
588	{
589	int rc = VERR_INTERNAL_ERROR_5;
590
591	/*
592	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
593	*/
594	uint32_t u32Family;
595	uint32_t u32Model;
596	uint32_t u32Stepping;
597	if (HMAmdIsSubjectToErratum170(&u32Family, &u32Model, &u32Stepping))
598	{
599	Log4(("SVMR0InitVM: AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
600	pVM->hm.s.svm.fAlwaysFlushTLB = true;
601	}
602
603	/*
604	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
605	*/
606	for (VMCPUID i = 0; i < pVM->cCpus; i++)
607	{
608	PVMCPU pVCpu = &pVM->aCpus[i];
609	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
610	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
611	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
612	}
613
614	for (VMCPUID i = 0; i < pVM->cCpus; i++)
615	{
616	PVMCPU pVCpu = &pVM->aCpus[i];
617
618	/*
619	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
620	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
621	*/
622	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
623	if (RT_FAILURE(rc))
624	goto failure_cleanup;
625
626	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
627	pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
628	Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
629	ASMMemZeroPage(pvVmcbHost);
630
631	/*
632	* Allocate one page for the guest-state VMCB.
633	*/
634	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
635	if (RT_FAILURE(rc))
636	goto failure_cleanup;
637
638	pVCpu->hm.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
639	pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
640	Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
641	ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
642
643	/*
644	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
645	* SVM to not require one.
646	*/
647	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
648	false /* fExecutable */);
649	if (RT_FAILURE(rc))
650	goto failure_cleanup;
651
652	pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
653	pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
654	/* Set all bits to intercept all MSR accesses (changed later on). */
655	ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
656	}
657
658	return VINF_SUCCESS;
659
660	failure_cleanup:
661	hmR0SvmFreeStructs(pVM);
662	return rc;
663	}
664
665
666	/**
667	* Does per-VM AMD-V termination.
668	*
669	* @returns VBox status code.
670	* @param pVM The cross context VM structure.
671	*/
672	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
673	{
674	hmR0SvmFreeStructs(pVM);
675	return VINF_SUCCESS;
676	}
677
678
679	/**
680	* Returns whether the VMCB Clean Bits feature is supported.
681	*
682	* @return @c true if supported, @c false otherwise.
683	* @param pVCpu The cross context virtual CPU structure.
684	* @param pCtx Pointer to the guest-CPU context.
685	*/
686	DECLINLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPU pVCpu, PCPUMCTX pCtx)
687	{
688	PVM pVM = pVCpu->CTX_SUFF(pVM);
689	#ifdef VBOX_WITH_NESTED_HWVIRT
690	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
691	{
692	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN)
693	&& pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
694	}
695	#else
696	RT_NOREF(pCtx);
697	#endif
698	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
699	}
700
701
702	/**
703	* Returns whether the decode assists feature is supported.
704	*
705	* @return @c true if supported, @c false otherwise.
706	* @param pVCpu The cross context virtual CPU structure.
707	* @param pCtx Pointer to the guest-CPU context.
708	*/
709	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPU pVCpu, PCPUMCTX pCtx)
710	{
711	PVM pVM = pVCpu->CTX_SUFF(pVM);
712	#ifdef VBOX_WITH_NESTED_HWVIRT
713	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
714	{
715	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
716	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
717	}
718	#else
719	RT_NOREF(pCtx);
720	#endif
721	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
722	}
723
724
725	/**
726	* Returns whether the NRIP_SAVE feature is supported.
727	*
728	* @return @c true if supported, @c false otherwise.
729	* @param pVCpu The cross context virtual CPU structure.
730	* @param pCtx Pointer to the guest-CPU context.
731	*/
732	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPU pVCpu, PCPUMCTX pCtx)
733	{
734	PVM pVM = pVCpu->CTX_SUFF(pVM);
735	#ifdef VBOX_WITH_NESTED_HWVIRT
736	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
737	{
738	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
739	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
740	}
741	#else
742	RT_NOREF(pCtx);
743	#endif
744	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
745	}
746
747
748	/**
749	* Sets the permission bits for the specified MSR in the MSRPM.
750	*
751	* @param pVmcb Pointer to the VM control block.
752	* @param pbMsrBitmap Pointer to the MSR bitmap.
753	* @param uMsr The MSR for which the access permissions are being set.
754	* @param enmRead MSR read permissions.
755	* @param enmWrite MSR write permissions.
756	*/
757	static void hmR0SvmSetMsrPermission(PSVMVMCB pVmcb, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
758	SVMMSREXITWRITE enmWrite)
759	{
760	uint16_t offMsrpm;
761	uint32_t uMsrpmBit;
762	int rc = HMSvmGetMsrpmOffsetAndBit(uMsr, &offMsrpm, &uMsrpmBit);
763	AssertRC(rc);
764
765	Assert(uMsrpmBit < 0x3fff);
766	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
767
768	pbMsrBitmap += offMsrpm;
769	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
770	ASMBitSet(pbMsrBitmap, uMsrpmBit);
771	else
772	ASMBitClear(pbMsrBitmap, uMsrpmBit);
773
774	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
775	ASMBitSet(pbMsrBitmap, uMsrpmBit + 1);
776	else
777	ASMBitClear(pbMsrBitmap, uMsrpmBit + 1);
778
779	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
780	}
781
782
783	/**
784	* Sets up AMD-V for the specified VM.
785	* This function is only called once per-VM during initalization.
786	*
787	* @returns VBox status code.
788	* @param pVM The cross context VM structure.
789	*/
790	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
791	{
792	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
793	AssertReturn(pVM, VERR_INVALID_PARAMETER);
794	Assert(pVM->hm.s.svm.fSupported);
795
796	bool const fPauseFilter = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
797	bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
798	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter && pVM->hm.s.svm.cPauseFilterThresholdTicks;
799
800	for (VMCPUID i = 0; i < pVM->cCpus; i++)
801	{
802	PVMCPU pVCpu = &pVM->aCpus[i];
803	PSVMVMCB pVmcb = pVM->aCpus[i].hm.s.svm.pVmcb;
804
805	AssertMsgReturn(pVmcb, ("Invalid pVmcb for vcpu[%u]\n", i), VERR_SVM_INVALID_PVMCB);
806
807	/* Initialize the #VMEXIT history array with end-of-array markers (UINT16_MAX). */
808	Assert(!pVCpu->hm.s.idxExitHistoryFree);
809	HMCPU_EXIT_HISTORY_RESET(pVCpu);
810
811	/* Always trap #AC for reasons of security. */
812	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
813
814	/* Always trap #DB for reasons of security. */
815	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
816
817	/* Trap exceptions unconditionally (debug purposes). */
818	#ifdef HMSVM_ALWAYS_TRAP_PF
819	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
820	#endif
821	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
822	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
823	pVmcb->ctrl.u32InterceptXcpt \|= 0
824	\| RT_BIT(X86_XCPT_BP)
825	\| RT_BIT(X86_XCPT_DE)
826	\| RT_BIT(X86_XCPT_NM)
827	\| RT_BIT(X86_XCPT_UD)
828	\| RT_BIT(X86_XCPT_NP)
829	\| RT_BIT(X86_XCPT_SS)
830	\| RT_BIT(X86_XCPT_GP)
831	\| RT_BIT(X86_XCPT_PF)
832	\| RT_BIT(X86_XCPT_MF)
833	;
834	#endif
835
836	/* Set up unconditional intercepts and conditions. */
837	pVmcb->ctrl.u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
838
839	/* CR0, CR4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
840	pVmcb->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
841
842	/* CR0, CR4 writes must be intercepted for the same reasons as above. */
843	pVmcb->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4);
844
845	/* Intercept all DRx reads and writes by default. Changed later on. */
846	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
847	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
848
849	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
850	pVmcb->ctrl.IntCtrl.n.u1VIntrMasking = 1;
851
852	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
853	and we currently deliver both PIC and APIC interrupts alike. See hmR0SvmInjectPendingEvent() */
854	pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
855
856	/* Set IO and MSR bitmap permission bitmap physical addresses. */
857	pVmcb->ctrl.u64IOPMPhysAddr = g_HCPhysIOBitmap;
858	pVmcb->ctrl.u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
859
860	/* No LBR virtualization. */
861	Assert(pVmcb->ctrl.u1LbrVirt == 0);
862
863	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
864	Assert(pVmcb->ctrl.u32VmcbCleanBits == 0);
865
866	/* The host ASID MBZ, for the guest start with 1. */
867	pVmcb->ctrl.TLBCtrl.n.u32ASID = 1;
868
869	/*
870	* Setup the PAT MSR (applicable for Nested Paging only).
871	* The default value should be 0x0007040600070406ULL, but we want to treat all guest memory as WB,
872	* so choose type 6 for all PAT slots.
873	*/
874	pVmcb->guest.u64GPAT = UINT64_C(0x0006060606060606);
875
876	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
877	pVmcb->ctrl.u1NestedPaging = pVM->hm.s.fNestedPaging;
878
879	/* Without Nested Paging, we need additionally intercepts. */
880	if (!pVM->hm.s.fNestedPaging)
881	{
882	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
883	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(3);
884	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(3);
885
886	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
887	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
888	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
889
890	/* Page faults must be intercepted to implement shadow paging. */
891	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
892	}
893
894	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
895	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
896	#endif
897
898	/* Apply the exceptions intercepts needed by the GIM provider. */
899	if (pVCpu->hm.s.fGIMTrapXcptUD)
900	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
901
902	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
903	if (fUsePauseFilter)
904	{
905	pVmcb->ctrl.u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
906	if (fPauseFilterThreshold)
907	pVmcb->ctrl.u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
908	}
909
910	/*
911	* The following MSRs are saved/restored automatically during the world-switch.
912	* Don't intercept guest read/write accesses to these MSRs.
913	*/
914	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
915	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
916	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
917	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
918	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
919	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
920	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
921	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
922	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
923	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
924	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
925	}
926
927	return VINF_SUCCESS;
928	}
929
930
931	/**
932	* Gets a pointer to the currently active guest or nested-guest VMCB.
933	*
934	* @returns Pointer to the current context VMCB.
935	* @param pVCpu The cross context virtual CPU structure.
936	* @param pCtx Pointer to the guest-CPU context.
937	*/
938	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
939	{
940	#ifdef VBOX_WITH_NESTED_HWVIRT
941	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
942	return pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
943	#else
944	RT_NOREF(pCtx);
945	#endif
946	return pVCpu->hm.s.svm.pVmcb;
947	}
948
949
950	/**
951	* Invalidates a guest page by guest virtual address.
952	*
953	* @returns VBox status code.
954	* @param pVM The cross context VM structure.
955	* @param pVCpu The cross context virtual CPU structure.
956	* @param GCVirt Guest virtual address of the page to invalidate.
957	*/
958	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
959	{
960	AssertReturn(pVM, VERR_INVALID_PARAMETER);
961	Assert(pVM->hm.s.svm.fSupported);
962
963	bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB \|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
964
965	/* Skip it if a TLB flush is already pending. */
966	if (!fFlushPending)
967	{
968	Log4(("SVMR0InvalidatePage %RGv\n", GCVirt));
969
970	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
971	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
972	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
973
974	#if HC_ARCH_BITS == 32
975	/* If we get a flush in 64-bit guest mode, then force a full TLB flush. INVLPGA takes only 32-bit addresses. */
976	if (CPUMIsGuestInLongMode(pVCpu))
977	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
978	else
979	#endif
980	{
981	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
982	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
983	}
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Flushes the appropriate tagged-TLB entries.
991	*
992	* @param pVCpu The cross context virtual CPU structure.
993	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
994	* @param pVmcb Pointer to the VM control block.
995	*/
996	static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
997	{
998	#ifndef VBOX_WITH_NESTED_HWVIRT
999	RT_NOREF(pCtx);
1000	#endif
1001
1002	PVM pVM = pVCpu->CTX_SUFF(pVM);
1003	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
1004
1005	/*
1006	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1007	* This can happen both for start & resume due to long jumps back to ring-3.
1008	*
1009	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no correlation
1010	* between it and the physical CPU.
1011	*
1012	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
1013	* so we cannot reuse the ASIDs without flushing.
1014	*/
1015	bool fNewAsid = false;
1016	Assert(pCpu->idCpu != NIL_RTCPUID);
1017	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1018	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes
1019	#ifdef VBOX_WITH_NESTED_HWVIRT
1020	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
1021	#endif
1022	)
1023	{
1024	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1025	pVCpu->hm.s.fForceTLBFlush = true;
1026	fNewAsid = true;
1027	}
1028
1029	/* Set TLB flush state as checked until we return from the world switch. */
1030	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1031
1032	/* Check for explicit TLB flushes. */
1033	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1034	{
1035	pVCpu->hm.s.fForceTLBFlush = true;
1036	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1037	}
1038
1039	/*
1040	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1041	* This Host CPU requirement takes precedence.
1042	*/
1043	if (pVM->hm.s.svm.fAlwaysFlushTLB)
1044	{
1045	pCpu->uCurrentAsid = 1;
1046	pVCpu->hm.s.uCurrentAsid = 1;
1047	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1048	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1049
1050	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1051	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1052
1053	/* Keep track of last CPU ID even when flushing all the time. */
1054	if (fNewAsid)
1055	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1056	}
1057	else
1058	{
1059	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1060	if (pVCpu->hm.s.fForceTLBFlush)
1061	{
1062	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1063	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1064
1065	if (fNewAsid)
1066	{
1067	++pCpu->uCurrentAsid;
1068
1069	bool fHitASIDLimit = false;
1070	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1071	{
1072	pCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1073	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1074	fHitASIDLimit = true;
1075	}
1076
1077	if ( fHitASIDLimit
1078	\|\| pCpu->fFlushAsidBeforeUse)
1079	{
1080	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1081	pCpu->fFlushAsidBeforeUse = false;
1082	}
1083
1084	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1085	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1086	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1087	}
1088	else
1089	{
1090	if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1091	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1092	else
1093	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1094	}
1095
1096	pVCpu->hm.s.fForceTLBFlush = false;
1097	}
1098	}
1099
1100	/* Update VMCB with the ASID. */
1101	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
1102	{
1103	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
1104	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1105	}
1106
1107	AssertMsg(pVCpu->hm.s.idLastCpu == pCpu->idCpu,
1108	("vcpu idLastCpu=%u pcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pCpu->idCpu));
1109	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1110	("Flush count mismatch for cpu %u (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1111	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1112	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
1113	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1114	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1115
1116	#ifdef VBOX_WITH_STATISTICS
1117	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1118	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1119	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1120	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1121	{
1122	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1123	}
1124	else
1125	{
1126	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1127	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1128	}
1129	#endif
1130	}
1131
1132
1133	/** @name 64-bit guest on 32-bit host OS helper functions.
1134	*
1135	* The host CPU is still 64-bit capable but the host OS is running in 32-bit
1136	* mode (code segment, paging). These wrappers/helpers perform the necessary
1137	* bits for the 32->64 switcher.
1138	*
1139	* @{ */
1140	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1141	/**
1142	* Prepares for and executes VMRUN (64-bit guests on a 32-bit host).
1143	*
1144	* @returns VBox status code.
1145	* @param HCPhysVmcbHost Physical address of host VMCB.
1146	* @param HCPhysVmcb Physical address of the VMCB.
1147	* @param pCtx Pointer to the guest-CPU context.
1148	* @param pVM The cross context VM structure.
1149	* @param pVCpu The cross context virtual CPU structure.
1150	*/
1151	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1152	{
1153	uint32_t aParam[8];
1154	aParam[0] = RT_LO_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Lo. */
1155	aParam[1] = RT_HI_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Hi. */
1156	aParam[2] = RT_LO_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Lo. */
1157	aParam[3] = RT_HI_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Hi. */
1158	aParam[4] = VM_RC_ADDR(pVM, pVM);
1159	aParam[5] = 0;
1160	aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1161	aParam[7] = 0;
1162
1163	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1164	}
1165
1166
1167	/**
1168	* Executes the specified VMRUN handler in 64-bit mode.
1169	*
1170	* @returns VBox status code.
1171	* @param pVM The cross context VM structure.
1172	* @param pVCpu The cross context virtual CPU structure.
1173	* @param pCtx Pointer to the guest-CPU context.
1174	* @param enmOp The operation to perform.
1175	* @param cParams Number of parameters.
1176	* @param paParam Array of 32-bit parameters.
1177	*/
1178	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
1179	uint32_t cParams, uint32_t *paParam)
1180	{
1181	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
1182	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
1183
1184	NOREF(pCtx);
1185
1186	/* Disable interrupts. */
1187	RTHCUINTREG uOldEFlags = ASMIntDisableFlags();
1188
1189	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1190	RTCPUID idHostCpu = RTMpCpuId();
1191	CPUMR0SetLApic(pVCpu, idHostCpu);
1192	#endif
1193
1194	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1195	CPUMSetHyperEIP(pVCpu, enmOp);
1196	for (int i = (int)cParams - 1; i >= 0; i--)
1197	CPUMPushHyper(pVCpu, paParam[i]);
1198
1199	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1200	/* Call the switcher. */
1201	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
1202	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1203
1204	/* Restore interrupts. */
1205	ASMSetFlags(uOldEFlags);
1206	return rc;
1207	}
1208
1209	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
1210	/** @} */
1211
1212
1213	/**
1214	* Adds an exception to the intercept exception bitmap in the VMCB and updates
1215	* the corresponding VMCB Clean bit.
1216	*
1217	* @param pVmcb Pointer to the VM control block.
1218	* @param u32Xcpt The value of the exception (X86_XCPT_*).
1219	*/
1220	DECLINLINE(void) hmR0SvmAddXcptIntercept(PSVMVMCB pVmcb, uint32_t u32Xcpt)
1221	{
1222	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(u32Xcpt)))
1223	{
1224	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(u32Xcpt);
1225	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1226	}
1227	}
1228
1229
1230	/**
1231	* Removes an exception from the intercept-exception bitmap in the VMCB and
1232	* updates the corresponding VMCB Clean bit.
1233	*
1234	* @param pVCpu The cross context virtual CPU structure.
1235	* @param pCtx Pointer to the guest-CPU context.
1236	* @param pVmcb Pointer to the VM control block.
1237	* @param u32Xcpt The value of the exception (X86_XCPT_*).
1238	*
1239	* @remarks This takes into account if we're executing a nested-guest and only
1240	* removes the exception intercept if both the guest -and- nested-guest
1241	* are not intercepting it.
1242	*/
1243	DECLINLINE(void) hmR0SvmRemoveXcptIntercept(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb, uint32_t u32Xcpt)
1244	{
1245	Assert(u32Xcpt != X86_XCPT_DB);
1246	Assert(u32Xcpt != X86_XCPT_AC);
1247	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1248	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(u32Xcpt))
1249	{
1250	bool fRemoveXcpt = true;
1251	#ifdef VBOX_WITH_NESTED_HWVIRT
1252	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1253	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1254	{
1255	Assert(pCtx->hwvirt.svm.fHMCachedVmcb); NOREF(pCtx);
1256	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
1257	fRemoveXcpt = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(u32Xcpt));
1258	}
1259	#else
1260	RT_NOREF2(pVCpu, pCtx);
1261	#endif
1262	if (fRemoveXcpt)
1263	{
1264	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(u32Xcpt);
1265	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1266	}
1267	}
1268	#else
1269	RT_NOREF3(pVCpu, pCtx, pVmcb);
1270	#endif
1271	}
1272
1273
1274	/**
1275	* Loads the guest (or nested-guest) CR0 control register into the guest-state
1276	* area in the VMCB.
1277	*
1278	* Although the guest CR0 is a separate field in the VMCB we have to consider
1279	* the FPU state itself which is shared between the host and the guest.
1280	*
1281	* @returns VBox status code.
1282	* @param pVCpu The cross context virtual CPU structure.
1283	* @param pVmcb Pointer to the VM control block.
1284	* @param pCtx Pointer to the guest-CPU context.
1285	*
1286	* @remarks No-long-jump zone!!!
1287	*/
1288	static void hmR0SvmLoadSharedCR0(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1289	{
1290	uint64_t u64GuestCR0 = pCtx->cr0;
1291
1292	/* Always enable caching. */
1293	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1294
1295	/*
1296	* When Nested Paging is not available use shadow page tables and intercept #PFs (the latter done in SVMR0SetupVM()).
1297	*/
1298	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1299	{
1300	u64GuestCR0 \|= X86_CR0_PG; /* When Nested Paging is not available, use shadow page tables. */
1301	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1302	}
1303
1304	/*
1305	* Guest FPU bits.
1306	*/
1307	bool fInterceptNM = false;
1308	bool fInterceptMF = false;
1309	u64GuestCR0 \|= X86_CR0_NE; /* Use internal x87 FPU exceptions handling rather than external interrupts. */
1310	if (CPUMIsGuestFPUStateActive(pVCpu))
1311	{
1312	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1313	if (!(pCtx->cr0 & X86_CR0_NE))
1314	{
1315	Log4(("hmR0SvmLoadSharedCR0: Intercepting Guest CR0.MP Old-style FPU handling!!!\n"));
1316	fInterceptMF = true;
1317	}
1318	}
1319	else
1320	{
1321	fInterceptNM = true; /* Guest FPU inactive, #VMEXIT on #NM for lazy FPU loading. */
1322	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
1323	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
1324	}
1325
1326	/*
1327	* Update the exception intercept bitmap.
1328	*/
1329	if (fInterceptNM)
1330	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_NM);
1331	else
1332	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_NM);
1333
1334	if (fInterceptMF)
1335	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_MF);
1336	else
1337	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_MF);
1338
1339	pVmcb->guest.u64CR0 = u64GuestCR0;
1340	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1341	}
1342
1343
1344	/**
1345	* Loads the guest/nested-guest control registers (CR2, CR3, CR4) into the VMCB.
1346	*
1347	* @returns VBox status code.
1348	* @param pVCpu The cross context virtual CPU structure.
1349	* @param pVmcb Pointer to the VM control block.
1350	* @param pCtx Pointer to the guest-CPU context.
1351	*
1352	* @remarks No-long-jump zone!!!
1353	*/
1354	static int hmR0SvmLoadGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1355	{
1356	PVM pVM = pVCpu->CTX_SUFF(pVM);
1357
1358	/*
1359	* Guest CR2.
1360	*/
1361	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR2))
1362	{
1363	pVmcb->guest.u64CR2 = pCtx->cr2;
1364	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1365	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
1366	}
1367
1368	/*
1369	* Guest CR3.
1370	*/
1371	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
1372	{
1373	if (pVM->hm.s.fNestedPaging)
1374	{
1375	PGMMODE enmShwPagingMode;
1376	#if HC_ARCH_BITS == 32
1377	if (CPUMIsGuestInLongModeEx(pCtx))
1378	enmShwPagingMode = PGMMODE_AMD64_NX;
1379	else
1380	#endif
1381	enmShwPagingMode = PGMGetHostMode(pVM);
1382
1383	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1384	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1385	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1386	pVmcb->guest.u64CR3 = pCtx->cr3;
1387	}
1388	else
1389	{
1390	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1391	Log4(("hmR0SvmLoadGuestControlRegs: CR3=%#RX64 (HyperCR3=%#RX64)\n", pCtx->cr3, pVmcb->guest.u64CR3));
1392	}
1393
1394	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1395	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
1396	}
1397
1398	/*
1399	* Guest CR4.
1400	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
1401	*/
1402	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
1403	{
1404	uint64_t u64GuestCR4 = pCtx->cr4;
1405	Assert(RT_HI_U32(u64GuestCR4) == 0);
1406	if (!pVM->hm.s.fNestedPaging)
1407	{
1408	switch (pVCpu->hm.s.enmShadowMode)
1409	{
1410	case PGMMODE_REAL:
1411	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1412	AssertFailed();
1413	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1414
1415	case PGMMODE_32_BIT: /* 32-bit paging. */
1416	u64GuestCR4 &= ~X86_CR4_PAE;
1417	break;
1418
1419	case PGMMODE_PAE: /* PAE paging. */
1420	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1421	/** Must use PAE paging as we could use physical memory > 4 GB */
1422	u64GuestCR4 \|= X86_CR4_PAE;
1423	break;
1424
1425	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1426	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1427	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1428	break;
1429	#else
1430	AssertFailed();
1431	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1432	#endif
1433
1434	default: /* shut up gcc */
1435	AssertFailed();
1436	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1437	}
1438	}
1439
1440	pVmcb->guest.u64CR4 = u64GuestCR4;
1441	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1442
1443	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1444	pVCpu->hm.s.fLoadSaveGuestXcr0 = (u64GuestCR4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1445
1446	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
1447	}
1448
1449	return VINF_SUCCESS;
1450	}
1451
1452
1453	/**
1454	* Loads the guest segment registers into the VMCB.
1455	*
1456	* @returns VBox status code.
1457	* @param pVCpu The cross context virtual CPU structure.
1458	* @param pVmcb Pointer to the VM control block.
1459	* @param pCtx Pointer to the guest-CPU context.
1460	*
1461	* @remarks No-long-jump zone!!!
1462	*/
1463	static void hmR0SvmLoadGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1464	{
1465	/* Guest Segment registers: CS, SS, DS, ES, FS, GS. */
1466	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
1467	{
1468	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1469	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1470	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1471	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1472	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1473	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1474
1475	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1476	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1477	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
1478	}
1479
1480	/* Guest TR. */
1481	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
1482	{
1483	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1484	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
1485	}
1486
1487	/* Guest LDTR. */
1488	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
1489	{
1490	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1491	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
1492	}
1493
1494	/* Guest GDTR. */
1495	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
1496	{
1497	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1498	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1499	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1500	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
1501	}
1502
1503	/* Guest IDTR. */
1504	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
1505	{
1506	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1507	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1508	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1509	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
1510	}
1511	}
1512
1513
1514	/**
1515	* Loads the guest MSRs into the VMCB.
1516	*
1517	* @param pVCpu The cross context virtual CPU structure.
1518	* @param pVmcb Pointer to the VM control block.
1519	* @param pCtx Pointer to the guest-CPU context.
1520	*
1521	* @remarks No-long-jump zone!!!
1522	*/
1523	static void hmR0SvmLoadGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1524	{
1525	/* Guest Sysenter MSRs. */
1526	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1527	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1528	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1529
1530	/*
1531	* Guest EFER MSR.
1532	* AMD-V requires guest EFER.SVME to be set. Weird.
1533	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1534	*/
1535	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
1536	{
1537	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1538	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1539	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
1540	}
1541
1542	/* 64-bit MSRs. */
1543	if (CPUMIsGuestInLongModeEx(pCtx))
1544	{
1545	pVmcb->guest.FS.u64Base = pCtx->fs.u64Base;
1546	pVmcb->guest.GS.u64Base = pCtx->gs.u64Base;
1547	}
1548	else
1549	{
1550	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit from guest EFER otherwise AMD-V expects amd64 shadow paging. */
1551	if (pCtx->msrEFER & MSR_K6_EFER_LME)
1552	{
1553	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1554	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1555	}
1556	}
1557
1558	/** @todo The following are used in 64-bit only (SYSCALL/SYSRET) but they might
1559	* be writable in 32-bit mode. Clarify with AMD spec. */
1560	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1561	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1562	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1563	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1564	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1565	}
1566
1567
1568	/**
1569	* Loads the guest (or nested-guest) debug state into the VMCB and programs the
1570	* necessary intercepts accordingly.
1571	*
1572	* @param pVCpu The cross context virtual CPU structure.
1573	* @param pVmcb Pointer to the VM control block.
1574	* @param pCtx Pointer to the guest-CPU context.
1575	*
1576	* @remarks No-long-jump zone!!!
1577	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1578	*/
1579	static void hmR0SvmLoadSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1580	{
1581	bool fInterceptMovDRx = false;
1582
1583	/*
1584	* Anyone single stepping on the host side? If so, we'll have to use the
1585	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1586	* the VMM level like the VT-x implementations does.
1587	*/
1588	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1589	if (fStepping)
1590	{
1591	pVCpu->hm.s.fClearTrapFlag = true;
1592	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1593	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1594	}
1595
1596	if ( fStepping
1597	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1598	{
1599	/*
1600	* Use the combined guest and host DRx values found in the hypervisor
1601	* register set because the debugger has breakpoints active or someone
1602	* is single stepping on the host side.
1603	*
1604	* Note! DBGF expects a clean DR6 state before executing guest code.
1605	*/
1606	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1607	if ( CPUMIsGuestInLongModeEx(pCtx)
1608	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
1609	{
1610	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1611	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1612	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1613	}
1614	else
1615	#endif
1616	if (!CPUMIsHyperDebugStateActive(pVCpu))
1617	{
1618	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1619	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1620	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1621	}
1622
1623	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1624	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1625	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1626	{
1627	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1628	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1629	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1630	pVCpu->hm.s.fUsingHyperDR7 = true;
1631	}
1632
1633	/** @todo If we cared, we could optimize to allow the guest to read registers
1634	* with the same values. */
1635	fInterceptMovDRx = true;
1636	Log5(("hmR0SvmLoadSharedDebugState: Loaded hyper DRx\n"));
1637	}
1638	else
1639	{
1640	/*
1641	* Update DR6, DR7 with the guest values if necessary.
1642	*/
1643	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1644	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1645	{
1646	pVmcb->guest.u64DR7 = pCtx->dr[7];
1647	pVmcb->guest.u64DR6 = pCtx->dr[6];
1648	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1649	pVCpu->hm.s.fUsingHyperDR7 = false;
1650	}
1651
1652	/*
1653	* If the guest has enabled debug registers, we need to load them prior to
1654	* executing guest code so they'll trigger at the right time.
1655	*/
1656	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1657	{
1658	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1659	if ( CPUMIsGuestInLongModeEx(pCtx)
1660	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
1661	{
1662	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1663	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1664	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
1665	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
1666	}
1667	else
1668	#endif
1669	if (!CPUMIsGuestDebugStateActive(pVCpu))
1670	{
1671	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1672	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1673	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1674	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1675	}
1676	Log5(("hmR0SvmLoadSharedDebugState: Loaded guest DRx\n"));
1677	}
1678	/*
1679	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
1680	* intercept #DB as DR6 is updated in the VMCB.
1681	*
1682	* Note! If we cared and dared, we could skip intercepting \#DB here.
1683	* However, \#DB shouldn't be performance critical, so we'll play safe
1684	* and keep the code similar to the VT-x code and always intercept it.
1685	*/
1686	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1687	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
1688	&& !CPUMIsGuestDebugStateActive(pVCpu))
1689	#else
1690	else if (!CPUMIsGuestDebugStateActive(pVCpu))
1691	#endif
1692	{
1693	fInterceptMovDRx = true;
1694	}
1695	}
1696
1697	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
1698	if (fInterceptMovDRx)
1699	{
1700	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1701	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1702	{
1703	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
1704	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
1705	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1706	}
1707	}
1708	else
1709	{
1710	if ( pVmcb->ctrl.u16InterceptRdDRx
1711	\|\| pVmcb->ctrl.u16InterceptWrDRx)
1712	{
1713	pVmcb->ctrl.u16InterceptRdDRx = 0;
1714	pVmcb->ctrl.u16InterceptWrDRx = 0;
1715	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1716	}
1717	}
1718	Log4(("hmR0SvmLoadSharedDebugState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
1719	}
1720
1721
1722	#ifdef VBOX_WITH_NESTED_HWVIRT
1723	/**
1724	* Loads the nested-guest APIC state (currently just the TPR).
1725	*
1726	* @param pVCpu The cross context virtual CPU structure.
1727	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
1728	*/
1729	static void hmR0SvmLoadGuestApicStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst)
1730	{
1731	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1732	{
1733	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
1734	pVmcbNstGst->ctrl.IntCtrl.n.u1VIntrMasking = 1;
1735	pVCpu->hm.s.svm.fSyncVTpr = false;
1736	pVmcbNstGst->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_TPR;
1737
1738	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1739	}
1740	}
1741	#endif
1742
1743	/**
1744	* Loads the guest APIC state (currently just the TPR).
1745	*
1746	* @returns VBox status code.
1747	* @param pVCpu The cross context virtual CPU structure.
1748	* @param pVmcb Pointer to the VM control block.
1749	* @param pCtx Pointer to the guest-CPU context.
1750	*/
1751	static int hmR0SvmLoadGuestApicState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1752	{
1753	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1754	return VINF_SUCCESS;
1755
1756	int rc = VINF_SUCCESS;
1757	PVM pVM = pVCpu->CTX_SUFF(pVM);
1758	if ( PDMHasApic(pVM)
1759	&& APICIsEnabled(pVCpu))
1760	{
1761	bool fPendingIntr;
1762	uint8_t u8Tpr;
1763	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
1764	AssertRCReturn(rc, rc);
1765
1766	/* Assume that we need to trap all TPR accesses and thus need not check on
1767	every #VMEXIT if we should update the TPR. */
1768	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
1769	pVCpu->hm.s.svm.fSyncVTpr = false;
1770
1771	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
1772	if (pVM->hm.s.fTPRPatchingActive)
1773	{
1774	pCtx->msrLSTAR = u8Tpr;
1775	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
1776
1777	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
1778	if (fPendingIntr)
1779	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
1780	else
1781	{
1782	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1783	pVCpu->hm.s.svm.fSyncVTpr = true;
1784	}
1785	}
1786	else
1787	{
1788	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
1789	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
1790
1791	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we can deliver the interrupt to the guest. */
1792	if (fPendingIntr)
1793	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
1794	else
1795	{
1796	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
1797	pVCpu->hm.s.svm.fSyncVTpr = true;
1798	}
1799
1800	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_TPR);
1801	}
1802	}
1803
1804	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1805	return rc;
1806	}
1807
1808
1809	/**
1810	* Loads the exception interrupts required for guest (or nested-guest) execution in
1811	* the VMCB.
1812	*
1813	* @param pVCpu The cross context virtual CPU structure.
1814	* @param pVmcb Pointer to the VM control block.
1815	* @param pCtx Pointer to the guest-CPU context.
1816	*/
1817	static void hmR0SvmLoadGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1818	{
1819	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1820	{
1821	/* Trap #UD for GIM provider (e.g. for hypercalls). */
1822	if (pVCpu->hm.s.fGIMTrapXcptUD)
1823	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_UD);
1824	else
1825	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_UD);
1826
1827	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
1828	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
1829	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_BP);
1830	else
1831	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_BP);
1832
1833	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmLoadSharedCR0(). */
1834	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
1835	}
1836	}
1837
1838
1839	#ifdef VBOX_WITH_NESTED_HWVIRT
1840	/**
1841	* Loads the intercepts required for nested-guest execution in the VMCB.
1842	*
1843	* This merges the guest and nested-guest intercepts in a way that if the outer
1844	* guest intercepts an exception we need to intercept it in the nested-guest as
1845	* well and handle it accordingly.
1846	*
1847	* @param pVCpu The cross context virtual CPU structure.
1848	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
1849	* @param pCtx Pointer to the guest-CPU context.
1850	*/
1851	static void hmR0SvmLoadGuestXcptInterceptsNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst, PCPUMCTX pCtx)
1852	{
1853	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1854	{
1855	/* First, load the guest intercepts into the guest VMCB. */
1856	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
1857	Assert(!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR));
1858	hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
1859
1860	/* Next, merge the intercepts into the nested-guest VMCB. */
1861	pVmcbNstGst->ctrl.u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
1862	pVmcbNstGst->ctrl.u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
1863
1864	/* Always intercept CR0, CR4 reads and writes as we alter them. */
1865	pVmcbNstGst->ctrl.u16InterceptRdCRx \|= RT_BIT(0) \| RT_BIT(4);
1866	pVmcbNstGst->ctrl.u16InterceptWrCRx \|= RT_BIT(0) \| RT_BIT(4);
1867
1868	/* Always intercept CR3 reads and writes without nested-paging as we load shadow page tables. */
1869	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1870	{
1871	pVmcbNstGst->ctrl.u16InterceptRdCRx \|= RT_BIT(3);
1872	pVmcbNstGst->ctrl.u16InterceptWrCRx \|= RT_BIT(3);
1873	}
1874
1875	/** @todo Figure out debugging with nested-guests, till then just intercept
1876	* all DR[0-15] accesses. */
1877	pVmcbNstGst->ctrl.u16InterceptRdDRx \|= 0xffff;
1878	pVmcbNstGst->ctrl.u16InterceptWrDRx \|= 0xffff;
1879
1880	pVmcbNstGst->ctrl.u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
1881	pVmcbNstGst->ctrl.u64InterceptCtrl \|= pVmcb->ctrl.u64InterceptCtrl
1882	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
1883
1884	/*
1885	* Remove control intercepts that we don't need while executing the nested-guest.
1886	*
1887	* VMMCALL when not intercepted raises a \#UD exception in the guest. However,
1888	* other SVM instructions like VMSAVE when not intercept can cause havoc on the
1889	* host as they can write to any location in physical memory, hence they always
1890	* need to be intercepted (they are included in HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS).
1891	*/
1892	Assert( (pVmcbNstGst->ctrl.u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
1893	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
1894	pVmcbNstGst->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_VMMCALL;
1895
1896	/* Finally, update the VMCB clean bits. */
1897	pVmcbNstGst->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1898
1899	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS));
1900	}
1901	}
1902	#endif
1903
1904
1905	/**
1906	* Sets up the appropriate function to run guest code.
1907	*
1908	* @returns VBox status code.
1909	* @param pVCpu The cross context virtual CPU structure.
1910	*
1911	* @remarks No-long-jump zone!!!
1912	*/
1913	static int hmR0SvmSetupVMRunHandler(PVMCPU pVCpu)
1914	{
1915	if (CPUMIsGuestInLongMode(pVCpu))
1916	{
1917	#ifndef VBOX_ENABLE_64_BITS_GUESTS
1918	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1919	#endif
1920	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
1921	#if HC_ARCH_BITS == 32
1922	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
1923	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
1924	#else
1925	/* 64-bit host or hybrid host. */
1926	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
1927	#endif
1928	}
1929	else
1930	{
1931	/* Guest is not in long mode, use the 32-bit handler. */
1932	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
1933	}
1934	return VINF_SUCCESS;
1935	}
1936
1937
1938	/**
1939	* Enters the AMD-V session.
1940	*
1941	* @returns VBox status code.
1942	* @param pVM The cross context VM structure.
1943	* @param pVCpu The cross context virtual CPU structure.
1944	* @param pCpu Pointer to the CPU info struct.
1945	*/
1946	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1947	{
1948	AssertPtr(pVM);
1949	AssertPtr(pVCpu);
1950	Assert(pVM->hm.s.svm.fSupported);
1951	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1952	NOREF(pVM); NOREF(pCpu);
1953
1954	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1955	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
1956
1957	pVCpu->hm.s.fLeaveDone = false;
1958	return VINF_SUCCESS;
1959	}
1960
1961
1962	/**
1963	* Thread-context callback for AMD-V.
1964	*
1965	* @param enmEvent The thread-context event.
1966	* @param pVCpu The cross context virtual CPU structure.
1967	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
1968	* @thread EMT(pVCpu)
1969	*/
1970	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
1971	{
1972	NOREF(fGlobalInit);
1973
1974	switch (enmEvent)
1975	{
1976	case RTTHREADCTXEVENT_OUT:
1977	{
1978	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1979	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
1980	VMCPU_ASSERT_EMT(pVCpu);
1981
1982	/* No longjmps (log-flush, locks) in this fragile context. */
1983	VMMRZCallRing3Disable(pVCpu);
1984
1985	if (!pVCpu->hm.s.fLeaveDone)
1986	{
1987	hmR0SvmLeave(pVCpu);
1988	pVCpu->hm.s.fLeaveDone = true;
1989	}
1990
1991	/* Leave HM context, takes care of local init (term). */
1992	int rc = HMR0LeaveCpu(pVCpu);
1993	AssertRC(rc); NOREF(rc);
1994
1995	/* Restore longjmp state. */
1996	VMMRZCallRing3Enable(pVCpu);
1997	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
1998	break;
1999	}
2000
2001	case RTTHREADCTXEVENT_IN:
2002	{
2003	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2004	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2005	VMCPU_ASSERT_EMT(pVCpu);
2006
2007	/* No longjmps (log-flush, locks) in this fragile context. */
2008	VMMRZCallRing3Disable(pVCpu);
2009
2010	/*
2011	* Initialize the bare minimum state required for HM. This takes care of
2012	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2013	*/
2014	int rc = HMR0EnterCpu(pVCpu);
2015	AssertRC(rc); NOREF(rc);
2016	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
2017
2018	pVCpu->hm.s.fLeaveDone = false;
2019
2020	/* Restore longjmp state. */
2021	VMMRZCallRing3Enable(pVCpu);
2022	break;
2023	}
2024
2025	default:
2026	break;
2027	}
2028	}
2029
2030
2031	/**
2032	* Saves the host state.
2033	*
2034	* @returns VBox status code.
2035	* @param pVM The cross context VM structure.
2036	* @param pVCpu The cross context virtual CPU structure.
2037	*
2038	* @remarks No-long-jump zone!!!
2039	*/
2040	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
2041	{
2042	NOREF(pVM);
2043	NOREF(pVCpu);
2044	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2045	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
2046	return VINF_SUCCESS;
2047	}
2048
2049
2050	/**
2051	* Loads the guest state into the VMCB.
2052	*
2053	* The CPU state will be loaded from these fields on every successful VM-entry.
2054	* Also sets up the appropriate VMRUN function to execute guest code based on
2055	* the guest CPU mode.
2056	*
2057	* @returns VBox status code.
2058	* @param pVM The cross context VM structure.
2059	* @param pVCpu The cross context virtual CPU structure.
2060	* @param pCtx Pointer to the guest-CPU context.
2061	*
2062	* @remarks No-long-jump zone!!!
2063	*/
2064	static int hmR0SvmLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2065	{
2066	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
2067
2068	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2069	AssertMsgReturn(pVmcb, ("Invalid pVmcb\n"), VERR_SVM_INVALID_PVMCB);
2070
2071	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2072
2073	int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcb, pCtx);
2074	AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestControlRegs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2075
2076	hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcb, pCtx);
2077	hmR0SvmLoadGuestMsrs(pVCpu, pVmcb, pCtx);
2078
2079	pVmcb->guest.u64RIP = pCtx->rip;
2080	pVmcb->guest.u64RSP = pCtx->rsp;
2081	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2082	pVmcb->guest.u64RAX = pCtx->rax;
2083
2084	rc = hmR0SvmLoadGuestApicState(pVCpu, pVmcb, pCtx);
2085	AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2086
2087	hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
2088
2089	rc = hmR0SvmSetupVMRunHandler(pVCpu);
2090	AssertLogRelMsgRCReturn(rc, ("hmR0SvmSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2091
2092	/* Clear any unused and reserved bits. */
2093	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2094	\| HM_CHANGED_GUEST_RSP
2095	\| HM_CHANGED_GUEST_RFLAGS
2096	\| HM_CHANGED_GUEST_SYSENTER_CS_MSR
2097	\| HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2098	\| HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2099	\| HM_CHANGED_GUEST_LAZY_MSRS /* Unused. */
2100	\| HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2101	\| HM_CHANGED_SVM_RESERVED2
2102	\| HM_CHANGED_SVM_RESERVED3
2103	\| HM_CHANGED_SVM_RESERVED4);
2104
2105	/* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2106	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2107	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
2108	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2109
2110	Log4(("hmR0SvmLoadGuestState: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 CR4=%#RX32\n", pCtx->cs.Sel, pCtx->rip,
2111	pCtx->eflags.u, pCtx->cr0, pCtx->cr3, pCtx->cr4));
2112	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2113	return rc;
2114	}
2115
2116
2117	#ifdef VBOX_WITH_NESTED_HWVIRT
2118	/**
2119	* Caches the nested-guest VMCB fields before we modify them for execution using
2120	* hardware-assisted SVM.
2121	*
2122	* @returns true if the VMCB was previously already cached, false otherwise.
2123	* @param pCtx Pointer to the guest-CPU context.
2124	*
2125	* @sa HMSvmNstGstVmExitNotify.
2126	*/
2127	static bool hmR0SvmVmRunCacheVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
2128	{
2129	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2130	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2131	PCSVMVMCBSTATESAVE pVmcbNstGstState = &pVmcbNstGst->guest;
2132	PSVMNESTEDVMCBCACHE pNstGstVmcbCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2133
2134	/*
2135	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2136	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2137	*
2138	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2139	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2140	*/
2141	bool const fWasCached = pCtx->hwvirt.svm.fHMCachedVmcb;
2142	if (!fWasCached)
2143	{
2144	pNstGstVmcbCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2145	pNstGstVmcbCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2146	pNstGstVmcbCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2147	pNstGstVmcbCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2148	pNstGstVmcbCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2149	pNstGstVmcbCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2150	pNstGstVmcbCache->u64CR0 = pVmcbNstGstState->u64CR0;
2151	pNstGstVmcbCache->u64CR3 = pVmcbNstGstState->u64CR3;
2152	pNstGstVmcbCache->u64CR4 = pVmcbNstGstState->u64CR4;
2153	pNstGstVmcbCache->u64EFER = pVmcbNstGstState->u64EFER;
2154	pNstGstVmcbCache->u64IOPMPhysAddr = pVmcbNstGstCtrl->u64IOPMPhysAddr;
2155	pNstGstVmcbCache->u64MSRPMPhysAddr = pVmcbNstGstCtrl->u64MSRPMPhysAddr;
2156	pNstGstVmcbCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2157	pNstGstVmcbCache->u32VmcbCleanBits = pVmcbNstGstCtrl->u32VmcbCleanBits;
2158	pNstGstVmcbCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2159	pNstGstVmcbCache->TLBCtrl = pVmcbNstGstCtrl->TLBCtrl;
2160	pNstGstVmcbCache->u1NestedPaging = pVmcbNstGstCtrl->u1NestedPaging;
2161	pCtx->hwvirt.svm.fHMCachedVmcb = true;
2162	Log4(("hmR0SvmVmRunCacheVmcb: Cached VMCB fields\n"));
2163	}
2164
2165	return fWasCached;
2166	}
2167
2168
2169	/**
2170	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2171	*
2172	* @param pVCpu The cross context virtual CPU structure.
2173	* @param pCtx Pointer to the guest-CPU context.
2174	*/
2175	static void hmR0SvmVmRunSetupVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
2176	{
2177	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2178	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2179
2180	/*
2181	* First cache the nested-guest VMCB fields we may potentially modify.
2182	*/
2183	bool const fVmcbCached = hmR0SvmVmRunCacheVmcb(pVCpu, pCtx);
2184	if (!fVmcbCached)
2185	{
2186	/*
2187	* The IOPM of the nested-guest can be ignored because the the guest always
2188	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2189	* into the nested-guest one and swap it back on the #VMEXIT.
2190	*/
2191	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2192
2193	/*
2194	* Load the host-physical address into the MSRPM rather than the nested-guest
2195	* physical address (currently we trap all MSRs in the nested-guest).
2196	*/
2197	pVmcbNstGstCtrl->u64MSRPMPhysAddr = g_HCPhysNstGstMsrBitmap;
2198
2199	/*
2200	* Use the same nested-paging as the "outer" guest. We can't dynamically
2201	* switch off nested-paging suddenly while executing a VM (see assertion at the
2202	* end of Trap0eHandler in PGMAllBth.h).
2203	*/
2204	pVmcbNstGstCtrl->u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
2205	}
2206	else
2207	{
2208	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2209	Assert(pVmcbNstGstCtrl->u64MSRPMPhysAddr = g_HCPhysNstGstMsrBitmap);
2210	Assert(RT_BOOL(pVmcbNstGstCtrl->u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
2211	}
2212	}
2213
2214
2215	/**
2216	* Loads the nested-guest state into the VMCB.
2217	*
2218	* @returns VBox status code.
2219	* @param pVCpu The cross context virtual CPU structure.
2220	* @param pCtx Pointer to the guest-CPU context.
2221	*
2222	* @remarks No-long-jump zone!!!
2223	*/
2224	static int hmR0SvmLoadGuestStateNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2225	{
2226	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2227
2228	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2229	Assert(pVmcbNstGst);
2230
2231	hmR0SvmVmRunSetupVmcb(pVCpu, pCtx);
2232
2233	int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcbNstGst, pCtx);
2234	AssertRCReturn(rc, rc);
2235
2236	hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcbNstGst, pCtx);
2237	hmR0SvmLoadGuestMsrs(pVCpu, pVmcbNstGst, pCtx);
2238	hmR0SvmLoadGuestApicStateNested(pVCpu, pVmcbNstGst);
2239
2240	pVmcbNstGst->guest.u64RIP = pCtx->rip;
2241	pVmcbNstGst->guest.u64RSP = pCtx->rsp;
2242	pVmcbNstGst->guest.u64RFlags = pCtx->eflags.u32;
2243	pVmcbNstGst->guest.u64RAX = pCtx->rax;
2244
2245	hmR0SvmLoadGuestXcptInterceptsNested(pVCpu, pVmcbNstGst, pCtx);
2246
2247	rc = hmR0SvmSetupVMRunHandler(pVCpu);
2248	AssertRCReturn(rc, rc);
2249
2250	/* Clear any unused and reserved bits. */
2251	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2252	\| HM_CHANGED_GUEST_RSP
2253	\| HM_CHANGED_GUEST_RFLAGS
2254	\| HM_CHANGED_GUEST_SYSENTER_CS_MSR
2255	\| HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2256	\| HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2257	\| HM_CHANGED_GUEST_LAZY_MSRS /* Unused. */
2258	\| HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2259	\| HM_CHANGED_SVM_RESERVED2
2260	\| HM_CHANGED_SVM_RESERVED3
2261	\| HM_CHANGED_SVM_RESERVED4);
2262
2263	/* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2264	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2265	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
2266	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2267
2268	Log4(("hmR0SvmLoadGuestStateNested: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 (HyperCR3=%#RX64) CR4=%#RX32 "
2269	"ESP=%#RX32 EBP=%#RX32 rc=%d\n", pCtx->cs.Sel, pCtx->rip, pCtx->eflags.u, pCtx->cr0, pCtx->cr3,
2270	pVmcbNstGst->guest.u64CR3, pCtx->cr4, pCtx->esp, pCtx->ebp, rc));
2271	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2272
2273	return rc;
2274	}
2275	#endif
2276
2277
2278	/**
2279	* Loads the state shared between the host and guest or nested-guest into the
2280	* VMCB.
2281	*
2282	* @param pVCpu The cross context virtual CPU structure.
2283	* @param pVmcb Pointer to the VM control block.
2284	* @param pCtx Pointer to the guest-CPU context.
2285	*
2286	* @remarks No-long-jump zone!!!
2287	*/
2288	static void hmR0SvmLoadSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
2289	{
2290	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2291	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2292
2293	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
2294	{
2295	hmR0SvmLoadSharedCR0(pVCpu, pVmcb, pCtx);
2296	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
2297	}
2298
2299	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
2300	{
2301	/** @todo Figure out stepping with nested-guest. */
2302	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2303	hmR0SvmLoadSharedDebugState(pVCpu, pVmcb, pCtx);
2304	else
2305	{
2306	pVmcb->guest.u64DR6 = pCtx->dr[6];
2307	pVmcb->guest.u64DR7 = pCtx->dr[7];
2308	Log4(("hmR0SvmLoadSharedState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2309	}
2310
2311	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
2312	}
2313
2314	/* Unused on AMD-V. */
2315	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
2316
2317	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
2318	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2319	}
2320
2321
2322	/**
2323	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU context.
2324	*
2325	* Currently there is no residual state left in the CPU that is not updated in the
2326	* VMCB.
2327	*
2328	* @returns VBox status code.
2329	* @param pVCpu The cross context virtual CPU structure.
2330	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2331	* out-of-sync. Make sure to update the required fields
2332	* before using them.
2333	* @param pVmcb Pointer to the VM control block.
2334	*/
2335	static void hmR0SvmSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PCSVMVMCB pVmcb)
2336	{
2337	Assert(VMMRZCallRing3IsEnabled(pVCpu));
2338
2339	pMixedCtx->rip = pVmcb->guest.u64RIP;
2340	pMixedCtx->rsp = pVmcb->guest.u64RSP;
2341	pMixedCtx->eflags.u32 = pVmcb->guest.u64RFlags;
2342	pMixedCtx->rax = pVmcb->guest.u64RAX;
2343
2344	/*
2345	* Guest interrupt shadow.
2346	*/
2347	if (pVmcb->ctrl.u1IntShadow)
2348	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
2349	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2350	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2351
2352	/*
2353	* Guest Control registers: CR2, CR3 (handled at the end) - accesses to other control registers are always intercepted.
2354	*/
2355	pMixedCtx->cr2 = pVmcb->guest.u64CR2;
2356
2357	/*
2358	* Guest MSRs.
2359	*/
2360	pMixedCtx->msrSTAR = pVmcb->guest.u64STAR; /* legacy syscall eip, cs & ss */
2361	pMixedCtx->msrLSTAR = pVmcb->guest.u64LSTAR; /* 64-bit mode syscall rip */
2362	pMixedCtx->msrCSTAR = pVmcb->guest.u64CSTAR; /* compatibility mode syscall rip */
2363	pMixedCtx->msrSFMASK = pVmcb->guest.u64SFMASK; /* syscall flag mask */
2364	pMixedCtx->msrKERNELGSBASE = pVmcb->guest.u64KernelGSBase; /* swapgs exchange value */
2365	pMixedCtx->SysEnter.cs = pVmcb->guest.u64SysEnterCS;
2366	pMixedCtx->SysEnter.eip = pVmcb->guest.u64SysEnterEIP;
2367	pMixedCtx->SysEnter.esp = pVmcb->guest.u64SysEnterESP;
2368
2369	/*
2370	* Guest segment registers (includes FS, GS base MSRs for 64-bit guests).
2371	*/
2372	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, CS, cs);
2373	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, SS, ss);
2374	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, DS, ds);
2375	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, ES, es);
2376	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, FS, fs);
2377	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, GS, gs);
2378
2379	/*
2380	* Correct the hidden CS granularity bit. Haven't seen it being wrong in any other
2381	* register (yet).
2382	*/
2383	/** @todo SELM might need to be fixed as it too should not care about the
2384	* granularity bit. See @bugref{6785}. */
2385	if ( !pMixedCtx->cs.Attr.n.u1Granularity
2386	&& pMixedCtx->cs.Attr.n.u1Present
2387	&& pMixedCtx->cs.u32Limit > UINT32_C(0xfffff))
2388	{
2389	Assert((pMixedCtx->cs.u32Limit & 0xfff) == 0xfff);
2390	pMixedCtx->cs.Attr.n.u1Granularity = 1;
2391	}
2392
2393	#ifdef VBOX_STRICT
2394	# define HMSVM_ASSERT_SEG_GRANULARITY(reg) \
2395	AssertMsg( !pMixedCtx->reg.Attr.n.u1Present \
2396	\|\| ( pMixedCtx->reg.Attr.n.u1Granularity \
2397	? (pMixedCtx->reg.u32Limit & 0xfff) == 0xfff \
2398	: pMixedCtx->reg.u32Limit <= UINT32_C(0xfffff)), \
2399	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", pMixedCtx->reg.u32Limit, \
2400	pMixedCtx->reg.Attr.u, pMixedCtx->reg.u64Base))
2401
2402	HMSVM_ASSERT_SEG_GRANULARITY(cs);
2403	HMSVM_ASSERT_SEG_GRANULARITY(ss);
2404	HMSVM_ASSERT_SEG_GRANULARITY(ds);
2405	HMSVM_ASSERT_SEG_GRANULARITY(es);
2406	HMSVM_ASSERT_SEG_GRANULARITY(fs);
2407	HMSVM_ASSERT_SEG_GRANULARITY(gs);
2408
2409	# undef HMSVM_ASSERT_SEL_GRANULARITY
2410	#endif
2411
2412	/*
2413	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the VMCB and uses that
2414	* and thus it's possible that when the CPL changes during guest execution that the SS DPL
2415	* isn't updated by AMD-V. Observed on some AMD Fusion CPUs with 64-bit guests.
2416	* See AMD spec. 15.5.1 "Basic operation".
2417	*/
2418	Assert(!(pVmcb->guest.u8CPL & ~0x3));
2419	pMixedCtx->ss.Attr.n.u2Dpl = pVmcb->guest.u8CPL & 0x3;
2420
2421	/*
2422	* Guest TR.
2423	* Fixup TR attributes so it's compatible with Intel. Important when saved-states are used
2424	* between Intel and AMD. See @bugref{6208#c39}.
2425	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2426	*/
2427	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, TR, tr);
2428	if (pMixedCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2429	{
2430	if ( pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2431	\|\| CPUMIsGuestInLongModeEx(pMixedCtx))
2432	pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2433	else if (pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2434	pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2435	}
2436
2437	/*
2438	* Guest Descriptor-Table registers.
2439	*/
2440	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, LDTR, ldtr);
2441	pMixedCtx->gdtr.cbGdt = pVmcb->guest.GDTR.u32Limit;
2442	pMixedCtx->gdtr.pGdt = pVmcb->guest.GDTR.u64Base;
2443
2444	pMixedCtx->idtr.cbIdt = pVmcb->guest.IDTR.u32Limit;
2445	pMixedCtx->idtr.pIdt = pVmcb->guest.IDTR.u64Base;
2446
2447	/*
2448	* Guest Debug registers.
2449	*/
2450	if (!pVCpu->hm.s.fUsingHyperDR7)
2451	{
2452	pMixedCtx->dr[6] = pVmcb->guest.u64DR6;
2453	pMixedCtx->dr[7] = pVmcb->guest.u64DR7;
2454	}
2455	else
2456	{
2457	Assert(pVmcb->guest.u64DR7 == CPUMGetHyperDR7(pVCpu));
2458	CPUMSetHyperDR6(pVCpu, pVmcb->guest.u64DR6);
2459	}
2460
2461	/*
2462	* With Nested Paging, CR3 changes are not intercepted. Therefore, sync. it now.
2463	* This is done as the very last step of syncing the guest state, as PGMUpdateCR3() may cause longjmp's to ring-3.
2464	*/
2465	if ( pVmcb->ctrl.u1NestedPaging
2466	&& pMixedCtx->cr3 != pVmcb->guest.u64CR3)
2467	{
2468	CPUMSetGuestCR3(pVCpu, pVmcb->guest.u64CR3);
2469	PGMUpdateCR3(pVCpu, pVmcb->guest.u64CR3);
2470	}
2471
2472	Log4(("hmR0SvmSaveGuestState: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 CR4=%#RX32\n", pMixedCtx->cs.Sel,
2473	pMixedCtx->rip, pMixedCtx->eflags.u, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4));
2474	}
2475
2476
2477	/**
2478	* Does the necessary state syncing before returning to ring-3 for any reason
2479	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2480	*
2481	* @param pVCpu The cross context virtual CPU structure.
2482	*
2483	* @remarks No-long-jmp zone!!!
2484	*/
2485	static void hmR0SvmLeave(PVMCPU pVCpu)
2486	{
2487	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2488	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2489	Assert(VMMR0IsLogFlushDisabled(pVCpu));
2490
2491	/*
2492	* !!! IMPORTANT !!!
2493	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2494	*/
2495
2496	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
2497	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
2498	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0); /** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2499
2500	/*
2501	* Restore host debug registers if necessary and resync on next R0 reentry.
2502	*/
2503	#ifdef VBOX_STRICT
2504	if (CPUMIsHyperDebugStateActive(pVCpu))
2505	{
2506	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
2507	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2508	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2509	}
2510	#endif
2511	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */))
2512	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);/** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2513
2514	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2515	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2516
2517	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2518	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
2519	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2520	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2521	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2522
2523	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
2524	}
2525
2526
2527	/**
2528	* Leaves the AMD-V session.
2529	*
2530	* @returns VBox status code.
2531	* @param pVCpu The cross context virtual CPU structure.
2532	*/
2533	static int hmR0SvmLeaveSession(PVMCPU pVCpu)
2534	{
2535	HM_DISABLE_PREEMPT();
2536	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2537	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2538
2539	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
2540	and done this from the SVMR0ThreadCtxCallback(). */
2541	if (!pVCpu->hm.s.fLeaveDone)
2542	{
2543	hmR0SvmLeave(pVCpu);
2544	pVCpu->hm.s.fLeaveDone = true;
2545	}
2546
2547	/*
2548	* !!! IMPORTANT !!!
2549	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2550	*/
2551
2552	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2553	/* Deregister hook now that we've left HM context before re-enabling preemption. */
2554	VMMR0ThreadCtxHookDisable(pVCpu);
2555
2556	/* Leave HM context. This takes care of local init (term). */
2557	int rc = HMR0LeaveCpu(pVCpu);
2558
2559	HM_RESTORE_PREEMPT();
2560	return rc;
2561	}
2562
2563
2564	/**
2565	* Does the necessary state syncing before doing a longjmp to ring-3.
2566	*
2567	* @returns VBox status code.
2568	* @param pVCpu The cross context virtual CPU structure.
2569	*
2570	* @remarks No-long-jmp zone!!!
2571	*/
2572	static int hmR0SvmLongJmpToRing3(PVMCPU pVCpu)
2573	{
2574	return hmR0SvmLeaveSession(pVCpu);
2575	}
2576
2577
2578	/**
2579	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
2580	* any remaining host state) before we longjump to ring-3 and possibly get
2581	* preempted.
2582	*
2583	* @param pVCpu The cross context virtual CPU structure.
2584	* @param enmOperation The operation causing the ring-3 longjump.
2585	* @param pvUser The user argument (pointer to the possibly
2586	* out-of-date guest-CPU context).
2587	*/
2588	static DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
2589	{
2590	RT_NOREF_PV(pvUser);
2591
2592	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
2593	{
2594	/*
2595	* !!! IMPORTANT !!!
2596	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
2597	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
2598	*/
2599	VMMRZCallRing3RemoveNotification(pVCpu);
2600	VMMRZCallRing3Disable(pVCpu);
2601	HM_DISABLE_PREEMPT();
2602
2603	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2604	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2605
2606	/* Restore host debug registers if necessary and resync on next R0 reentry. */
2607	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2608
2609	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
2610	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2611	VMMR0ThreadCtxHookDisable(pVCpu);
2612
2613	/* Leave HM context. This takes care of local init (term). */
2614	HMR0LeaveCpu(pVCpu);
2615
2616	HM_RESTORE_PREEMPT();
2617	return VINF_SUCCESS;
2618	}
2619
2620	Assert(pVCpu);
2621	Assert(pvUser);
2622	Assert(VMMRZCallRing3IsEnabled(pVCpu));
2623	HMSVM_ASSERT_PREEMPT_SAFE();
2624
2625	VMMRZCallRing3Disable(pVCpu);
2626	Assert(VMMR0IsLogFlushDisabled(pVCpu));
2627
2628	Log4(("hmR0SvmCallRing3Callback->hmR0SvmLongJmpToRing3\n"));
2629	int rc = hmR0SvmLongJmpToRing3(pVCpu);
2630	AssertRCReturn(rc, rc);
2631
2632	VMMRZCallRing3Enable(pVCpu);
2633	return VINF_SUCCESS;
2634	}
2635
2636
2637	/**
2638	* Take necessary actions before going back to ring-3.
2639	*
2640	* An action requires us to go back to ring-3. This function does the necessary
2641	* steps before we can safely return to ring-3. This is not the same as longjmps
2642	* to ring-3, this is voluntary.
2643	*
2644	* @returns VBox status code.
2645	* @param pVM The cross context VM structure.
2646	* @param pVCpu The cross context virtual CPU structure.
2647	* @param pCtx Pointer to the guest-CPU context.
2648	* @param rcExit The reason for exiting to ring-3. Can be
2649	* VINF_VMM_UNKNOWN_RING3_CALL.
2650	*/
2651	static int hmR0SvmExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rcExit)
2652	{
2653	Assert(pVM);
2654	Assert(pVCpu);
2655	Assert(pCtx);
2656	HMSVM_ASSERT_PREEMPT_SAFE();
2657
2658	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
2659	VMMRZCallRing3Disable(pVCpu);
2660	Log4(("hmR0SvmExitToRing3: VCPU[%u]: rcExit=%d LocalFF=%#RX32 GlobalFF=%#RX32\n", pVCpu->idCpu, rcExit,
2661	pVCpu->fLocalForcedActions, pVM->fGlobalForcedActions));
2662
2663	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
2664	if (pVCpu->hm.s.Event.fPending)
2665	{
2666	hmR0SvmPendingEventToTrpmTrap(pVCpu);
2667	Assert(!pVCpu->hm.s.Event.fPending);
2668	}
2669
2670	/* Sync. the necessary state for going back to ring-3. */
2671	hmR0SvmLeaveSession(pVCpu);
2672	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2673
2674	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2675	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
2676	\| CPUM_CHANGED_LDTR
2677	\| CPUM_CHANGED_GDTR
2678	\| CPUM_CHANGED_IDTR
2679	\| CPUM_CHANGED_TR
2680	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
2681	if ( pVM->hm.s.fNestedPaging
2682	&& CPUMIsGuestPagingEnabledEx(pCtx))
2683	{
2684	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
2685	}
2686
2687	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
2688	if (rcExit != VINF_EM_RAW_INTERRUPT)
2689	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
2690
2691	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
2692
2693	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
2694	VMMRZCallRing3RemoveNotification(pVCpu);
2695	VMMRZCallRing3Enable(pVCpu);
2696
2697	/*
2698	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
2699	* and if we're injecting an event we should have a TRPM trap pending.
2700	*/
2701	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
2702	pVCpu->hm.s.u32HMError = rcExit,
2703	VERR_SVM_IPE_5);
2704	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
2705	pVCpu->hm.s.u32HMError = rcExit,
2706	VERR_SVM_IPE_4);
2707
2708	return rcExit;
2709	}
2710
2711
2712	#ifdef VBOX_WITH_NESTED_HWVIRT
2713	/**
2714	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
2715	* intercepts for the nested-guest.
2716	*
2717	* @param pVM The cross context VM structure.
2718	* @param pVCpu The cross context virtual CPU structure.
2719	* @param pCtx Pointer to the nested guest-CPU context.
2720	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2721	*
2722	* @remarks No-long-jump zone!!!
2723	*/
2724	static void hmR0SvmUpdateTscOffsettingNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcbNstGst)
2725	{
2726	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
2727
2728	bool fParavirtTsc;
2729	uint64_t uTscOffset;
2730	bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
2731
2732	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2733	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2734
2735	/*
2736	* Only avoid intercepting if we determined the host TSC (++) is stable enough
2737	* to not intercept -and- the nested-hypervisor itself does not want to intercept it.
2738	*/
2739	if ( fCanUseRealTsc
2740	&& !(pVmcbNstGstCache->u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
2741	{
2742	pVmcbNstGstCtrl->u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSC;
2743	pVmcbNstGstCtrl->u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSCP;
2744	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2745	}
2746	else
2747	{
2748	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSC;
2749	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSCP;
2750	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2751	}
2752
2753	/* Apply the nested-guest VMCB's TSC offset over the guest one. */
2754	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
2755
2756	/* Update the nested-guest VMCB with the combined TSC offset (of guest and nested-guest). */
2757	pVmcbNstGstCtrl->u64TSCOffset = uTscOffset;
2758
2759	/* Finally update the VMCB clean bits since we touched the intercepts as well as the TSC offset. */
2760	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2761
2762	if (fParavirtTsc)
2763	{
2764	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
2765	information before every VM-entry, hence disable it for performance sake. */
2766	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2767	}
2768	}
2769	#endif
2770
2771
2772	/**
2773	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
2774	* intercepts.
2775	*
2776	* @param pVM The cross context VM structure.
2777	* @param pVCpu The cross context virtual CPU structure.
2778	* @param pVmcb Pointer to the VM control block.
2779	*
2780	* @remarks No-long-jump zone!!!
2781	*/
2782	static void hmR0SvmUpdateTscOffsetting(PVM pVM, PVMCPU pVCpu, PSVMVMCB pVmcb)
2783	{
2784	bool fParavirtTsc;
2785	bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVmcb->ctrl.u64TSCOffset, &fParavirtTsc);
2786	if (fCanUseRealTsc)
2787	{
2788	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSC;
2789	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSCP;
2790	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2791	}
2792	else
2793	{
2794	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSC;
2795	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSCP;
2796	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2797	}
2798	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2799
2800	/** @todo later optimize this to be done elsewhere and not before every
2801	* VM-entry. */
2802	if (fParavirtTsc)
2803	{
2804	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
2805	information before every VM-entry, hence disable it for performance sake. */
2806	#if 0
2807	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
2808	AssertRC(rc);
2809	#endif
2810	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2811	}
2812	}
2813
2814
2815	/**
2816	* Sets an event as a pending event to be injected into the guest.
2817	*
2818	* @param pVCpu The cross context virtual CPU structure.
2819	* @param pEvent Pointer to the SVM event.
2820	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
2821	* page-fault.
2822	*
2823	* @remarks Statistics counter assumes this is a guest event being reflected to
2824	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
2825	*/
2826	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
2827	{
2828	Assert(!pVCpu->hm.s.Event.fPending);
2829	Assert(pEvent->n.u1Valid);
2830
2831	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
2832	pVCpu->hm.s.Event.fPending = true;
2833	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
2834
2835	Log4(("hmR0SvmSetPendingEvent: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
2836	pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
2837	}
2838
2839
2840	/**
2841	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
2842	*
2843	* @param pVCpu The cross context virtual CPU structure.
2844	*/
2845	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
2846	{
2847	SVMEVENT Event;
2848	Event.u = 0;
2849	Event.n.u1Valid = 1;
2850	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2851	Event.n.u8Vector = X86_XCPT_UD;
2852	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2853	}
2854
2855
2856	/**
2857	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
2858	*
2859	* @param pVCpu The cross context virtual CPU structure.
2860	*/
2861	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
2862	{
2863	SVMEVENT Event;
2864	Event.u = 0;
2865	Event.n.u1Valid = 1;
2866	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2867	Event.n.u8Vector = X86_XCPT_DB;
2868	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2869	}
2870
2871
2872	/**
2873	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
2874	*
2875	* @param pVCpu The cross context virtual CPU structure.
2876	* @param pCtx Pointer to the guest-CPU context.
2877	* @param u32ErrCode The error-code for the page-fault.
2878	* @param uFaultAddress The page fault address (CR2).
2879	*
2880	* @remarks This updates the guest CR2 with @a uFaultAddress!
2881	*/
2882	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
2883	{
2884	SVMEVENT Event;
2885	Event.u = 0;
2886	Event.n.u1Valid = 1;
2887	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2888	Event.n.u8Vector = X86_XCPT_PF;
2889	Event.n.u1ErrorCodeValid = 1;
2890	Event.n.u32ErrorCode = u32ErrCode;
2891
2892	/* Update CR2 of the guest. */
2893	if (pCtx->cr2 != uFaultAddress)
2894	{
2895	pCtx->cr2 = uFaultAddress;
2896	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
2897	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR2);
2898	}
2899
2900	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
2901	}
2902
2903
2904	/**
2905	* Sets a device-not-available (\#NM) exception as pending-for-injection into
2906	* the VM.
2907	*
2908	* @param pVCpu The cross context virtual CPU structure.
2909	*/
2910	DECLINLINE(void) hmR0SvmSetPendingXcptNM(PVMCPU pVCpu)
2911	{
2912	SVMEVENT Event;
2913	Event.u = 0;
2914	Event.n.u1Valid = 1;
2915	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2916	Event.n.u8Vector = X86_XCPT_NM;
2917	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2918	}
2919
2920
2921	/**
2922	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
2923	*
2924	* @param pVCpu The cross context virtual CPU structure.
2925	*/
2926	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
2927	{
2928	SVMEVENT Event;
2929	Event.u = 0;
2930	Event.n.u1Valid = 1;
2931	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2932	Event.n.u8Vector = X86_XCPT_MF;
2933	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2934	}
2935
2936
2937	/**
2938	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
2939	*
2940	* @param pVCpu The cross context virtual CPU structure.
2941	*/
2942	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
2943	{
2944	SVMEVENT Event;
2945	Event.u = 0;
2946	Event.n.u1Valid = 1;
2947	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2948	Event.n.u8Vector = X86_XCPT_DF;
2949	Event.n.u1ErrorCodeValid = 1;
2950	Event.n.u32ErrorCode = 0;
2951	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2952	}
2953
2954
2955	/**
2956	* Injects an event into the guest upon VMRUN by updating the relevant field
2957	* in the VMCB.
2958	*
2959	* @param pVCpu The cross context virtual CPU structure.
2960	* @param pVmcb Pointer to the guest VM control block.
2961	* @param pCtx Pointer to the guest-CPU context.
2962	* @param pEvent Pointer to the event.
2963	*
2964	* @remarks No-long-jump zone!!!
2965	* @remarks Requires CR0!
2966	*/
2967	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx, PSVMEVENT pEvent)
2968	{
2969	NOREF(pVCpu); NOREF(pCtx);
2970
2971	pVmcb->ctrl.EventInject.u = pEvent->u;
2972	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
2973
2974	Log4(("hmR0SvmInjectEventVmcb: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
2975	pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
2976	}
2977
2978
2979
2980	/**
2981	* Converts any TRPM trap into a pending HM event. This is typically used when
2982	* entering from ring-3 (not longjmp returns).
2983	*
2984	* @param pVCpu The cross context virtual CPU structure.
2985	*/
2986	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
2987	{
2988	Assert(TRPMHasTrap(pVCpu));
2989	Assert(!pVCpu->hm.s.Event.fPending);
2990
2991	uint8_t uVector;
2992	TRPMEVENT enmTrpmEvent;
2993	RTGCUINT uErrCode;
2994	RTGCUINTPTR GCPtrFaultAddress;
2995	uint8_t cbInstr;
2996
2997	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
2998	AssertRC(rc);
2999
3000	SVMEVENT Event;
3001	Event.u = 0;
3002	Event.n.u1Valid = 1;
3003	Event.n.u8Vector = uVector;
3004
3005	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3006	if (enmTrpmEvent == TRPM_TRAP)
3007	{
3008	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3009	switch (uVector)
3010	{
3011	case X86_XCPT_NMI:
3012	{
3013	Event.n.u3Type = SVM_EVENT_NMI;
3014	break;
3015	}
3016
3017	case X86_XCPT_PF:
3018	case X86_XCPT_DF:
3019	case X86_XCPT_TS:
3020	case X86_XCPT_NP:
3021	case X86_XCPT_SS:
3022	case X86_XCPT_GP:
3023	case X86_XCPT_AC:
3024	{
3025	Event.n.u1ErrorCodeValid = 1;
3026	Event.n.u32ErrorCode = uErrCode;
3027	break;
3028	}
3029	}
3030	}
3031	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3032	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3033	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3034	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3035	else
3036	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3037
3038	rc = TRPMResetTrap(pVCpu);
3039	AssertRC(rc);
3040
3041	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3042	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3043
3044	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3045	}
3046
3047
3048	/**
3049	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3050	* AMD-V to execute any instruction.
3051	*
3052	* @param pVCpu The cross context virtual CPU structure.
3053	*/
3054	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
3055	{
3056	Assert(pVCpu->hm.s.Event.fPending);
3057	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3058
3059	SVMEVENT Event;
3060	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3061
3062	uint8_t uVector = Event.n.u8Vector;
3063	uint8_t uVectorType = Event.n.u3Type;
3064	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event);
3065
3066	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, uVectorType));
3067
3068	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3069	AssertRC(rc);
3070
3071	if (Event.n.u1ErrorCodeValid)
3072	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3073
3074	if ( uVectorType == SVM_EVENT_EXCEPTION
3075	&& uVector == X86_XCPT_PF)
3076	{
3077	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3078	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3079	}
3080	else if (uVectorType == SVM_EVENT_SOFTWARE_INT)
3081	{
3082	AssertMsg( uVectorType == SVM_EVENT_SOFTWARE_INT
3083	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
3084	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
3085	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3086	}
3087	pVCpu->hm.s.Event.fPending = false;
3088	}
3089
3090
3091	/**
3092	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3093	* now.
3094	*
3095	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3096	* @param pVCpu The cross context virtual CPU structure.
3097	* @param pCtx Pointer to the guest-CPU context.
3098	*
3099	* @remarks No-long-jump zone!!!
3100	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3101	*/
3102	DECLINLINE(bool) hmR0SvmIsIntrShadowActive(PVMCPU pVCpu, PCPUMCTX pCtx)
3103	{
3104	/*
3105	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3106	* inhibit interrupts or clear any existing interrupt-inhibition.
3107	*/
3108	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3109	{
3110	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3111	{
3112	/*
3113	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3114	* AMD-V, the flag's condition to be cleared is met and thus the cleared state is correct.
3115	*/
3116	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3117	return false;
3118	}
3119	return true;
3120	}
3121	return false;
3122	}
3123
3124
3125	/**
3126	* Sets the virtual interrupt intercept control in the VMCB.
3127	*
3128	* @param pVmcb Pointer to the VM control block.
3129	*/
3130	DECLINLINE(void) hmR0SvmSetVirtIntrIntercept(PSVMVMCB pVmcb)
3131	{
3132	/*
3133	* When AVIC isn't supported, indicate that a virtual interrupt is pending and to
3134	* cause a #VMEXIT when the guest is ready to accept interrupts. At #VMEXIT, we
3135	* then get the interrupt from the APIC (updating ISR at the right time) and
3136	* inject the interrupt.
3137	*
3138	* With AVIC is supported, we could make use of the asynchronously delivery without
3139	* #VMEXIT and we would be passing the AVIC page to SVM.
3140	*/
3141	if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3142	{
3143	Assert(pVmcb->ctrl.IntCtrl.n.u1VIrqPending == 0);
3144	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3145	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VINTR;
3146	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_TPR);
3147	Log4(("Set VINTR intercept\n"));
3148	}
3149	}
3150
3151
3152	/**
3153	* Clears the virtual interrupt intercept control in the VMCB as
3154	* we are figured the guest is unable process any interrupts
3155	* at this point of time.
3156	*
3157	* @param pVmcb Pointer to the VM control block.
3158	*/
3159	DECLINLINE(void) hmR0SvmClearVirtIntrIntercept(PSVMVMCB pVmcb)
3160	{
3161	if (pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR)
3162	{
3163	Assert(pVmcb->ctrl.IntCtrl.n.u1VIrqPending == 1);
3164	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 0;
3165	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_VINTR;
3166	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_TPR);
3167	Log4(("Cleared VINTR intercept\n"));
3168	}
3169	}
3170
3171
3172	/**
3173	* Sets the IRET intercept control in the VMCB which instructs AMD-V to cause a
3174	* \#VMEXIT as soon as a guest starts executing an IRET. This is used to unblock
3175	* virtual NMIs.
3176	*
3177	* @param pVmcb Pointer to the VM control block.
3178	*/
3179	DECLINLINE(void) hmR0SvmSetIretIntercept(PSVMVMCB pVmcb)
3180	{
3181	if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_IRET))
3182	{
3183	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_IRET;
3184	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3185
3186	Log4(("Setting IRET intercept\n"));
3187	}
3188	}
3189
3190
3191	/**
3192	* Clears the IRET intercept control in the VMCB.
3193	*
3194	* @param pVmcb Pointer to the VM control block.
3195	*/
3196	DECLINLINE(void) hmR0SvmClearIretIntercept(PSVMVMCB pVmcb)
3197	{
3198	if (pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_IRET)
3199	{
3200	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_IRET;
3201	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS);
3202
3203	Log4(("Clearing IRET intercept\n"));
3204	}
3205	}
3206
3207	#ifdef VBOX_WITH_NESTED_HWVIRT
3208
3209
3210	/**
3211	* Evaluates the event to be delivered to the nested-guest and sets it as the
3212	* pending event.
3213	*
3214	* @returns VBox strict status code.
3215	* @param pVCpu The cross context virtual CPU structure.
3216	* @param pCtx Pointer to the guest-CPU context.
3217	*/
3218	static VBOXSTRICTRC hmR0SvmEvaluatePendingEventNested(PVMCPU pVCpu, PCPUMCTX pCtx)
3219	{
3220	Log4Func(("\n"));
3221
3222	Assert(!pVCpu->hm.s.Event.fPending);
3223
3224	bool const fGif = pCtx->hwvirt.svm.fGif;
3225	if (fGif)
3226	{
3227	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
3228
3229	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3230
3231	/*
3232	* Check if the nested-guest can receive NMIs.
3233	* NMIs are higher priority than regular interrupts.
3234	*/
3235	/** @todo SMI. SMIs take priority over NMIs. */
3236	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI))
3237	{
3238	bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3239	if (fBlockNmi)
3240	hmR0SvmSetIretIntercept(pVmcbNstGst);
3241	else if (fIntShadow)
3242	{
3243	/** @todo Figure this out, how we shall manage virt. intercept if the
3244	* nested-guest already has one set and/or if we really need it? */
3245	//hmR0SvmSetVirtIntrIntercept(pVmcbNstGst);
3246	}
3247	else
3248	{
3249	Log4(("Pending NMI\n"));
3250
3251	SVMEVENT Event;
3252	Event.u = 0;
3253	Event.n.u1Valid = 1;
3254	Event.n.u8Vector = X86_XCPT_NMI;
3255	Event.n.u3Type = SVM_EVENT_NMI;
3256
3257	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3258	hmR0SvmSetIretIntercept(pVmcbNstGst);
3259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3260	return VINF_SUCCESS;
3261	}
3262	}
3263
3264	/*
3265	* Check if the nested-guest can receive external interrupts (generated by
3266	* the guest's PIC/APIC).
3267	*
3268	* External intercepts, NMI, SMI etc. from the physical CPU are -always- intercepted
3269	* when executing using hardware-assisted SVM, see HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS.
3270	*
3271	* External interrupts that are generated for the outer guest may be intercepted
3272	* depending on how the nested-guest VMCB was programmed by guest software.
3273	*
3274	* Physical interrupts always take priority over virtual interrupts,
3275	* see AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3276	*/
3277	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
3278	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3279	&& !fIntShadow
3280	&& !pVCpu->hm.s.fSingleInstruction
3281	&& CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx))
3282	{
3283	if (pVmcbNstGstCache->u64InterceptCtrl & SVM_CTRL_INTERCEPT_INTR)
3284	{
3285	Log4(("Intercepting external interrupt -> #VMEXIT\n"));
3286	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3287	}
3288
3289	uint8_t u8Interrupt;
3290	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3291	if (RT_SUCCESS(rc))
3292	{
3293	Log4(("Injecting external interrupt u8Interrupt=%#x\n", u8Interrupt));
3294
3295	SVMEVENT Event;
3296	Event.u = 0;
3297	Event.n.u1Valid = 1;
3298	Event.n.u8Vector = u8Interrupt;
3299	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3300
3301	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3302	}
3303	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3304	{
3305	/*
3306	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3307	* updated eventually when the TPR is written by the guest.
3308	*/
3309	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3310	}
3311	else
3312	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3313	}
3314
3315	/*
3316	* Check if the nested-guest is intercepting virtual (using V_IRQ and related fields)
3317	* interrupt injection. The virtual interrupt injection itself, if any, will be done
3318	* by the physical CPU.
3319	*/
3320	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)
3321	&& (pVmcbNstGstCache->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR)
3322	&& CPUMCanSvmNstGstTakeVirtIntr(pCtx))
3323	{
3324	Log4(("Intercepting virtual interrupt -> #VMEXIT\n"));
3325	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_VINTR, 0, 0);
3326	}
3327	}
3328
3329	return VINF_SUCCESS;
3330	}
3331	#endif
3332
3333
3334	/**
3335	* Evaluates the event to be delivered to the guest and sets it as the pending
3336	* event.
3337	*
3338	* @param pVCpu The cross context virtual CPU structure.
3339	* @param pCtx Pointer to the guest-CPU context.
3340	*
3341	* @remarks Don't use this function when we are actively executing a
3342	* nested-guest, use hmR0SvmEvaluatePendingEventNested instead.
3343	*/
3344	static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
3345	{
3346	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3347	Assert(!pVCpu->hm.s.Event.fPending);
3348
3349	#ifdef VBOX_WITH_NESTED_HWVIRT
3350	bool const fGif = pCtx->hwvirt.svm.fGif;
3351	#else
3352	bool const fGif = true;
3353	#endif
3354	Log4Func(("fGif=%RTbool\n", fGif));
3355
3356	/*
3357	* If the global interrupt flag (GIF) isn't set, even NMIs and other events are blocked.
3358	* See AMD spec. Table 15-10. "Effect of the GIF on Interrupt Handling".
3359	*/
3360	if (fGif)
3361	{
3362	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3363	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3364	bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3365	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3366
3367	Log4Func(("fGif=%RTbool fBlockInt=%RTbool fIntShadow=%RTbool APIC/PIC_Pending=%RTbool\n", fGif, fBlockInt, fIntShadow,
3368	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
3369
3370	/** @todo SMI. SMIs take priority over NMIs. */
3371	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
3372	{
3373	if (fBlockNmi)
3374	hmR0SvmSetIretIntercept(pVmcb);
3375	else if (fIntShadow)
3376	hmR0SvmSetVirtIntrIntercept(pVmcb);
3377	else
3378	{
3379	Log4(("Pending NMI\n"));
3380
3381	SVMEVENT Event;
3382	Event.u = 0;
3383	Event.n.u1Valid = 1;
3384	Event.n.u8Vector = X86_XCPT_NMI;
3385	Event.n.u3Type = SVM_EVENT_NMI;
3386
3387	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3388	hmR0SvmSetIretIntercept(pVmcb);
3389	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3390	return;
3391	}
3392	}
3393	else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3394	&& !pVCpu->hm.s.fSingleInstruction)
3395	{
3396	/*
3397	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
3398	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
3399	*/
3400	if ( !fBlockInt
3401	&& !fIntShadow)
3402	{
3403	uint8_t u8Interrupt;
3404	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3405	if (RT_SUCCESS(rc))
3406	{
3407	Log4(("Injecting external interrupt u8Interrupt=%#x\n", u8Interrupt));
3408
3409	SVMEVENT Event;
3410	Event.u = 0;
3411	Event.n.u1Valid = 1;
3412	Event.n.u8Vector = u8Interrupt;
3413	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3414
3415	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3416	}
3417	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3418	{
3419	/*
3420	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3421	* updated eventually when the TPR is written by the guest.
3422	*/
3423	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3424	}
3425	else
3426	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3427	}
3428	else
3429	hmR0SvmSetVirtIntrIntercept(pVmcb);
3430	}
3431	}
3432	}
3433
3434
3435	/**
3436	* Injects any pending events into the guest or nested-guest.
3437	*
3438	* @param pVCpu The cross context virtual CPU structure.
3439	* @param pCtx Pointer to the guest-CPU context.
3440	* @param pVmcb Pointer to the VM control block.
3441	*/
3442	static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
3443	{
3444	Assert(!TRPMHasTrap(pVCpu));
3445	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3446
3447	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3448
3449	/*
3450	* When executing the nested-guest, we avoid assertions on whether the
3451	* event injection is valid purely based on EFLAGS, as V_INTR_MASKING
3452	* affects the interpretation of interruptibility (see CPUMCanSvmNstGstTakePhysIntr).
3453	*/
3454	#ifndef VBOX_WITH_NESTED_HWVIRT
3455	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3456	#endif
3457
3458	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
3459	{
3460	SVMEVENT Event;
3461	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3462	Assert(Event.n.u1Valid);
3463
3464	#ifndef VBOX_WITH_NESTED_HWVIRT
3465	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3466	{
3467	Assert(!fBlockInt);
3468	Assert(!fIntShadow);
3469	}
3470	else if (Event.n.u3Type == SVM_EVENT_NMI)
3471	Assert(!fIntShadow);
3472	NOREF(fBlockInt);
3473	#else
3474	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3475	#endif
3476
3477	Log4(("Injecting pending HM event\n"));
3478	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, pCtx, &Event);
3479	pVCpu->hm.s.Event.fPending = false;
3480
3481	#ifdef VBOX_WITH_STATISTICS
3482	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3483	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3484	else
3485	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3486	#endif
3487	}
3488
3489	/*
3490	* Update the guest interrupt shadow in the guest or nested-guest VMCB.
3491	*
3492	* For nested-guests: We need to update it too for the scenario where IEM executes
3493	* the nested-guest but execution later continues here with an interrupt shadow active.
3494	*/
3495	pVmcb->ctrl.u1IntShadow = fIntShadow;
3496	}
3497
3498
3499	/**
3500	* Reports world-switch error and dumps some useful debug info.
3501	*
3502	* @param pVM The cross context VM structure.
3503	* @param pVCpu The cross context virtual CPU structure.
3504	* @param rcVMRun The return code from VMRUN (or
3505	* VERR_SVM_INVALID_GUEST_STATE for invalid
3506	* guest-state).
3507	* @param pCtx Pointer to the guest-CPU context.
3508	*/
3509	static void hmR0SvmReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx)
3510	{
3511	NOREF(pCtx);
3512	HMSVM_ASSERT_PREEMPT_SAFE();
3513	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3514	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3515
3516	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3517	{
3518	hmR0DumpRegs(pVM, pVCpu, pCtx); NOREF(pVM);
3519	#ifdef VBOX_STRICT
3520	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3521	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3522	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3523	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3524	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3525	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3526	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3527	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3528	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3529	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3530
3531	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3532	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3533	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3534
3535	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3536	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3537	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3538	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved0));
3539	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3540	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3541	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3542	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3543	Log4(("ctrl.IntCtrl.u6Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved1));
3544	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3545	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3546
3547	Log4(("ctrl.u1IntShadow %#x\n", pVmcb->ctrl.u1IntShadow));
3548	Log4(("ctrl.u1GuestIntMask %#x\n", pVmcb->ctrl.u1GuestIntMask));
3549	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3550	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3551	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3552	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3553	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3554	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3555	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3556	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3557	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3558	Log4(("ctrl.u1NestedPaging %#x\n", pVmcb->ctrl.u1NestedPaging));
3559	Log4(("ctrl.u1Sev %#x\n", pVmcb->ctrl.u1Sev));
3560	Log4(("ctrl.u1SevEs %#x\n", pVmcb->ctrl.u1SevEs));
3561	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3562	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3563	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3564	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3565	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3566	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3567
3568	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3569
3570	Log4(("ctrl.u1Lbrvirt %#x\n", pVmcb->ctrl.u1LbrVirt));
3571	Log4(("ctrl.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.u1VirtVmsaveVmload));
3572
3573	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3574	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3575	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3576	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3577	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3578	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3579	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3580	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3581	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3582	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3583	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3584	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3585	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3586	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3587	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3588	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3589	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3590	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3591	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3592	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3593
3594	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3595	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3596
3597	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3598	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3599	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3600	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3601
3602	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3603	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3604
3605	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3606	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3607	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3608	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3609
3610	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3611	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3612	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3613	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3614	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3615	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3616	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3617
3618	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3619	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3620	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3621	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3622
3623	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3624	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3625	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3626
3627	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3628	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3629	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3630	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3631	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3632	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3633	Log4(("guest.u64GPAT %#RX64\n", pVmcb->guest.u64GPAT));
3634	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3635	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3636	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3637	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3638	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3639	#endif /* VBOX_STRICT */
3640	}
3641	else
3642	Log4(("hmR0SvmReportWorldSwitchError: rcVMRun=%d\n", rcVMRun));
3643
3644	NOREF(pVmcb);
3645	}
3646
3647
3648	/**
3649	* Check per-VM and per-VCPU force flag actions that require us to go back to
3650	* ring-3 for one reason or another.
3651	*
3652	* @returns VBox status code (information status code included).
3653	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3654	* ring-3.
3655	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3656	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3657	* interrupts)
3658	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3659	* all EMTs to be in ring-3.
3660	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3661	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3662	* to the EM loop.
3663	*
3664	* @param pVM The cross context VM structure.
3665	* @param pVCpu The cross context virtual CPU structure.
3666	* @param pCtx Pointer to the guest-CPU context.
3667	*/
3668	static int hmR0SvmCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3669	{
3670	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3671
3672	/* On AMD-V we don't need to update CR3, PAE PDPES lazily. See hmR0SvmSaveGuestState(). */
3673	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3674	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
3675
3676	/* Update pending interrupts into the APIC's IRR. */
3677	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3678	APICUpdatePendingInterrupts(pVCpu);
3679
3680	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
3681	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3682	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
3683	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3684	{
3685	/* Pending PGM C3 sync. */
3686	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3687	{
3688	int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3689	if (rc != VINF_SUCCESS)
3690	{
3691	Log4(("hmR0SvmCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3692	return rc;
3693	}
3694	}
3695
3696	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3697	/* -XXX- what was that about single stepping? */
3698	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
3699	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3700	{
3701	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3702	int rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3703	Log4(("hmR0SvmCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3704	return rc;
3705	}
3706
3707	/* Pending VM request packets, such as hardware interrupts. */
3708	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
3709	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
3710	{
3711	Log4(("hmR0SvmCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
3712	return VINF_EM_PENDING_REQUEST;
3713	}
3714
3715	/* Pending PGM pool flushes. */
3716	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3717	{
3718	Log4(("hmR0SvmCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
3719	return VINF_PGM_POOL_FLUSH_PENDING;
3720	}
3721
3722	/* Pending DMA requests. */
3723	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
3724	{
3725	Log4(("hmR0SvmCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
3726	return VINF_EM_RAW_TO_R3;
3727	}
3728	}
3729
3730	return VINF_SUCCESS;
3731	}
3732
3733
3734	#ifdef VBOX_WITH_NESTED_HWVIRT
3735	/**
3736	* Does the preparations before executing nested-guest code in AMD-V.
3737	*
3738	* @returns VBox status code (informational status codes included).
3739	* @retval VINF_SUCCESS if we can proceed with running the guest.
3740	* @retval VINF_* scheduling changes, we have to go back to ring-3.
3741	*
3742	* @param pVM The cross context VM structure.
3743	* @param pVCpu The cross context virtual CPU structure.
3744	* @param pCtx Pointer to the guest-CPU context.
3745	* @param pSvmTransient Pointer to the SVM transient structure.
3746	*
3747	* @remarks Same caveats regarding longjumps as hmR0SvmPreRunGuest applies.
3748	* @sa hmR0SvmPreRunGuest.
3749	*/
3750	static int hmR0SvmPreRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3751	{
3752	HMSVM_ASSERT_PREEMPT_SAFE();
3753
3754	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3755	{
3756	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
3757	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
3758	return VINF_EM_RESCHEDULE_REM;
3759	#endif
3760	}
3761	else
3762	return VINF_SVM_VMEXIT;
3763
3764	/* Check force flag actions that might require us to go back to ring-3. */
3765	int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
3766	if (rc != VINF_SUCCESS)
3767	return rc;
3768
3769	if (TRPMHasTrap(pVCpu))
3770	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
3771	else if (!pVCpu->hm.s.Event.fPending)
3772	{
3773	VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEventNested(pVCpu, pCtx);
3774	if (rcStrict != VINF_SUCCESS)
3775	return VBOXSTRICTRC_VAL(rcStrict);
3776	}
3777
3778	/*
3779	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
3780	* Just do it in software, see @bugref{8411}.
3781	* NB: If we could continue a task switch exit we wouldn't need to do this.
3782	*/
3783	if (RT_UNLIKELY( !pVM->hm.s.svm.u32Features
3784	&& pVCpu->hm.s.Event.fPending
3785	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
3786	{
3787	return VINF_EM_RAW_INJECT_TRPM_EVENT;
3788	}
3789
3790	/*
3791	* Load the nested-guest state.
3792	*/
3793	rc = hmR0SvmLoadGuestStateNested(pVCpu, pCtx);
3794	AssertRCReturn(rc, rc);
3795	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull); /** @todo Get new STAM counter for this? */
3796
3797	/* Ensure we've cached (and hopefully modified) the VMCB for execution using hardware SVM. */
3798	Assert(pCtx->hwvirt.svm.fHMCachedVmcb);
3799
3800	/*
3801	* No longjmps to ring-3 from this point on!!!
3802	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
3803	* This also disables flushing of the R0-logger instance (if any).
3804	*/
3805	VMMRZCallRing3Disable(pVCpu);
3806
3807	/*
3808	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
3809	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
3810	*
3811	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
3812	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
3813	*
3814	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
3815	* executing guest code.
3816	*/
3817	pSvmTransient->fEFlags = ASMIntDisableFlags();
3818	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
3819	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3820	{
3821	ASMSetFlags(pSvmTransient->fEFlags);
3822	VMMRZCallRing3Enable(pVCpu);
3823	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3824	return VINF_EM_RAW_TO_R3;
3825	}
3826	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3827	{
3828	ASMSetFlags(pSvmTransient->fEFlags);
3829	VMMRZCallRing3Enable(pVCpu);
3830	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3831	return VINF_EM_RAW_INTERRUPT;
3832	}
3833
3834	/*
3835	* If we are injecting an NMI, we must set VMCPU_FF_BLOCK_NMIS only when we are going to execute
3836	* guest code for certain (no exits to ring-3). Otherwise, we could re-read the flag on re-entry into
3837	* AMD-V and conclude that NMI inhibition is active when we have not even delivered the NMI.
3838	*
3839	* With VT-x, this is handled by the Guest interruptibility information VMCS field which will set the
3840	* VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
3841	*/
3842	if (pVCpu->hm.s.Event.fPending)
3843	{
3844	SVMEVENT Event;
3845	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3846	if ( Event.n.u1Valid
3847	&& Event.n.u3Type == SVM_EVENT_NMI
3848	&& Event.n.u8Vector == X86_XCPT_NMI
3849	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3850	{
3851	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3852	}
3853	}
3854
3855	return VINF_SUCCESS;
3856	}
3857	#endif
3858
3859
3860	/**
3861	* Does the preparations before executing guest code in AMD-V.
3862	*
3863	* This may cause longjmps to ring-3 and may even result in rescheduling to the
3864	* recompiler. We must be cautious what we do here regarding committing
3865	* guest-state information into the VMCB assuming we assuredly execute the guest
3866	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
3867	* clearing the common-state (TRPM/forceflags), we must undo those changes so
3868	* that the recompiler can (and should) use them when it resumes guest
3869	* execution. Otherwise such operations must be done when we can no longer
3870	* exit to ring-3.
3871	*
3872	* @returns VBox status code (informational status codes included).
3873	* @retval VINF_SUCCESS if we can proceed with running the guest.
3874	* @retval VINF_* scheduling changes, we have to go back to ring-3.
3875	*
3876	* @param pVM The cross context VM structure.
3877	* @param pVCpu The cross context virtual CPU structure.
3878	* @param pCtx Pointer to the guest-CPU context.
3879	* @param pSvmTransient Pointer to the SVM transient structure.
3880	*/
3881	static int hmR0SvmPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3882	{
3883	HMSVM_ASSERT_PREEMPT_SAFE();
3884	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3885
3886	/* Check force flag actions that might require us to go back to ring-3. */
3887	int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
3888	if (rc != VINF_SUCCESS)
3889	return rc;
3890
3891	if (TRPMHasTrap(pVCpu))
3892	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
3893	else if (!pVCpu->hm.s.Event.fPending)
3894	hmR0SvmEvaluatePendingEvent(pVCpu, pCtx);
3895
3896	/*
3897	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
3898	* Just do it in software, see @bugref{8411}.
3899	* NB: If we could continue a task switch exit we wouldn't need to do this.
3900	*/
3901	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending && (((pVCpu->hm.s.Event.u64IntInfo >> 8) & 7) == SVM_EVENT_NMI)))
3902	if (RT_UNLIKELY(!pVM->hm.s.svm.u32Features))
3903	return VINF_EM_RAW_INJECT_TRPM_EVENT;
3904
3905	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
3906	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
3907	#endif
3908
3909	/* Load the guest bits that are not shared with the host in any way since we can longjmp or get preempted. */
3910	rc = hmR0SvmLoadGuestState(pVM, pVCpu, pCtx);
3911	AssertRCReturn(rc, rc);
3912	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3913
3914	/*
3915	* If we're not intercepting TPR changes in the guest, save the guest TPR before the world-switch
3916	* so we can update it on the way back if the guest changed the TPR.
3917	*/
3918	if (pVCpu->hm.s.svm.fSyncVTpr)
3919	{
3920	if (pVM->hm.s.fTPRPatchingActive)
3921	pSvmTransient->u8GuestTpr = pCtx->msrLSTAR;
3922	else
3923	{
3924	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3925	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
3926	}
3927	}
3928
3929	/*
3930	* No longjmps to ring-3 from this point on!!!
3931	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
3932	* This also disables flushing of the R0-logger instance (if any).
3933	*/
3934	VMMRZCallRing3Disable(pVCpu);
3935
3936	/*
3937	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
3938	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
3939	*
3940	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
3941	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
3942	*
3943	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
3944	* executing guest code.
3945	*/
3946	pSvmTransient->fEFlags = ASMIntDisableFlags();
3947	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
3948	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3949	{
3950	ASMSetFlags(pSvmTransient->fEFlags);
3951	VMMRZCallRing3Enable(pVCpu);
3952	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3953	return VINF_EM_RAW_TO_R3;
3954	}
3955	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3956	{
3957	ASMSetFlags(pSvmTransient->fEFlags);
3958	VMMRZCallRing3Enable(pVCpu);
3959	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3960	return VINF_EM_RAW_INTERRUPT;
3961	}
3962
3963	/*
3964	* If we are injecting an NMI, we must set VMCPU_FF_BLOCK_NMIS only when we are going to execute
3965	* guest code for certain (no exits to ring-3). Otherwise, we could re-read the flag on re-entry into
3966	* AMD-V and conclude that NMI inhibition is active when we have not even delivered the NMI.
3967	*
3968	* With VT-x, this is handled by the Guest interruptibility information VMCS field which will set the
3969	* VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
3970	*/
3971	if (pVCpu->hm.s.Event.fPending)
3972	{
3973	SVMEVENT Event;
3974	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3975	if ( Event.n.u1Valid
3976	&& Event.n.u3Type == SVM_EVENT_NMI
3977	&& Event.n.u8Vector == X86_XCPT_NMI
3978	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3979	{
3980	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3981	}
3982	}
3983
3984	return VINF_SUCCESS;
3985	}
3986
3987
3988	#ifdef VBOX_WITH_NESTED_HWVIRT
3989	/**
3990	* Prepares to run nested-guest code in AMD-V and we've committed to doing so. This
3991	* means there is no backing out to ring-3 or anywhere else at this point.
3992	*
3993	* @param pVM The cross context VM structure.
3994	* @param pVCpu The cross context virtual CPU structure.
3995	* @param pCtx Pointer to the guest-CPU context.
3996	* @param pSvmTransient Pointer to the SVM transient structure.
3997	*
3998	* @remarks Called with preemption disabled.
3999	* @remarks No-long-jump zone!!!
4000	*/
4001	static void hmR0SvmPreRunGuestCommittedNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4002	{
4003	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4004	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4005	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4006	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4007
4008	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4009	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4010
4011	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4012	hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcbNstGst);
4013
4014	if ( pVCpu->hm.s.fPreloadGuestFpu
4015	&& !CPUMIsGuestFPUStateActive(pVCpu))
4016	{
4017	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4018	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4019	}
4020
4021	/* Load the state shared between host and nested-guest (FPU, debug). */
4022	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4023	hmR0SvmLoadSharedState(pVCpu, pVmcbNstGst, pCtx);
4024
4025	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4026	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4027
4028	/* Setup TSC offsetting. */
4029	RTCPUID idCurrentCpu = hmR0GetCurrentCpu()->idCpu;
4030	if ( pSvmTransient->fUpdateTscOffsetting
4031	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
4032	{
4033	hmR0SvmUpdateTscOffsettingNested(pVM, pVCpu, pCtx, pVmcbNstGst);
4034	pSvmTransient->fUpdateTscOffsetting = false;
4035	}
4036
4037	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4038	if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
4039	pVmcbNstGst->ctrl.u32VmcbCleanBits = 0;
4040
4041	/* Store status of the shared guest-host state at the time of VMRUN. */
4042	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4043	if (CPUMIsGuestInLongModeEx(pCtx))
4044	{
4045	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4046	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4047	}
4048	else
4049	#endif
4050	{
4051	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4052	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4053	}
4054	pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
4055
4056	/* The TLB flushing would've already been setup by the nested-hypervisor. */
4057	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4058	hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcbNstGst);
4059	Assert(hmR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
4060
4061	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4062
4063	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4064	to start executing. */
4065
4066	/*
4067	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4068	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4069	*
4070	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4071	*/
4072	uint8_t pbMsrBitmap = (uint8_t )pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4073	if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4074	&& !(pVmcbNstGst->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4075	{
4076	hmR0SvmSetMsrPermission(pVmcbNstGst, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4077	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4078	uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4079	if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
4080	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTscAux);
4081	pSvmTransient->fRestoreTscAuxMsr = true;
4082	}
4083	else
4084	{
4085	hmR0SvmSetMsrPermission(pVmcbNstGst, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4086	pSvmTransient->fRestoreTscAuxMsr = false;
4087	}
4088
4089	/*
4090	* If VMCB Clean bits isn't supported by the CPU or exposed by the guest,
4091	* mark all state-bits as dirty indicating to the CPU to re-load from VMCB.
4092	*/
4093	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4094	if (!fSupportsVmcbCleanBits)
4095	pVmcbNstGst->ctrl.u32VmcbCleanBits = 0;
4096	}
4097	#endif
4098
4099
4100	/**
4101	* Prepares to run guest code in AMD-V and we've committed to doing so. This
4102	* means there is no backing out to ring-3 or anywhere else at this
4103	* point.
4104	*
4105	* @param pVM The cross context VM structure.
4106	* @param pVCpu The cross context virtual CPU structure.
4107	* @param pCtx Pointer to the guest-CPU context.
4108	* @param pSvmTransient Pointer to the SVM transient structure.
4109	*
4110	* @remarks Called with preemption disabled.
4111	* @remarks No-long-jump zone!!!
4112	*/
4113	static void hmR0SvmPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4114	{
4115	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4116	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4117	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4118	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
4119
4120	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4121	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4122
4123	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4124	hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcb);
4125
4126	if ( pVCpu->hm.s.fPreloadGuestFpu
4127	&& !CPUMIsGuestFPUStateActive(pVCpu))
4128	{
4129	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4130	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4131	}
4132
4133	/* Load the state shared between host and guest (FPU, debug). */
4134	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4135	hmR0SvmLoadSharedState(pVCpu, pVmcb, pCtx);
4136
4137	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4138	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4139
4140	/* Setup TSC offsetting. */
4141	RTCPUID idCurrentCpu = hmR0GetCurrentCpu()->idCpu;
4142	if ( pSvmTransient->fUpdateTscOffsetting
4143	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
4144	{
4145	hmR0SvmUpdateTscOffsetting(pVM, pVCpu, pVmcb);
4146	pSvmTransient->fUpdateTscOffsetting = false;
4147	}
4148
4149	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4150	if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
4151	pVmcb->ctrl.u32VmcbCleanBits = 0;
4152
4153	/* Store status of the shared guest-host state at the time of VMRUN. */
4154	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4155	if (CPUMIsGuestInLongModeEx(pCtx))
4156	{
4157	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4158	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4159	}
4160	else
4161	#endif
4162	{
4163	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4164	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4165	}
4166	pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
4167
4168	/* Flush the appropriate tagged-TLB entries. */
4169	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4170	hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcb);
4171	Assert(hmR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
4172
4173	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4174
4175	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4176	to start executing. */
4177
4178	/*
4179	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4180	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4181	*
4182	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4183	*/
4184	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4185	if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4186	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4187	{
4188	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4189	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4190	uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4191	if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
4192	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTscAux);
4193	pSvmTransient->fRestoreTscAuxMsr = true;
4194	}
4195	else
4196	{
4197	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4198	pSvmTransient->fRestoreTscAuxMsr = false;
4199	}
4200
4201	/* If VMCB Clean bits isn't supported by the CPU, simply mark all state-bits as dirty, indicating (re)load-from-VMCB. */
4202	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4203	if (!fSupportsVmcbCleanBits)
4204	pVmcb->ctrl.u32VmcbCleanBits = 0;
4205	}
4206
4207
4208	/**
4209	* Wrapper for running the guest code in AMD-V.
4210	*
4211	* @returns VBox strict status code.
4212	* @param pVM The cross context VM structure.
4213	* @param pVCpu The cross context virtual CPU structure.
4214	* @param pCtx Pointer to the guest-CPU context.
4215	*
4216	* @remarks No-long-jump zone!!!
4217	*/
4218	DECLINLINE(int) hmR0SvmRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4219	{
4220	/*
4221	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4222	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4223	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4224	*/
4225	#ifdef VBOX_WITH_KERNEL_USING_XMM
4226	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4227	pVCpu->hm.s.svm.pfnVMRun);
4228	#else
4229	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4230	#endif
4231	}
4232
4233
4234	#ifdef VBOX_WITH_NESTED_HWVIRT
4235	/**
4236	* Wrapper for running the nested-guest code in AMD-V.
4237	*
4238	* @returns VBox strict status code.
4239	* @param pVM The cross context VM structure.
4240	* @param pVCpu The cross context virtual CPU structure.
4241	* @param pCtx Pointer to the guest-CPU context.
4242	*
4243	* @remarks No-long-jump zone!!!
4244	*/
4245	DECLINLINE(int) hmR0SvmRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4246	{
4247	/*
4248	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4249	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4250	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4251	*/
4252	#ifdef VBOX_WITH_KERNEL_USING_XMM
4253	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4254	pVCpu->hm.s.svm.pfnVMRun);
4255	#else
4256	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4257	#endif
4258	}
4259
4260
4261	/**
4262	* Performs some essential restoration of state after running nested-guest code in
4263	* AMD-V.
4264	*
4265	* @param pVM The cross context VM structure.
4266	* @param pVCpu The cross context virtual CPU structure.
4267	* @param pMixedCtx Pointer to the nested-guest-CPU context. The data maybe
4268	* out-of-sync. Make sure to update the required fields
4269	* before using them.
4270	* @param pSvmTransient Pointer to the SVM transient structure.
4271	* @param rcVMRun Return code of VMRUN.
4272	*
4273	* @remarks Called with interrupts disabled.
4274	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4275	* unconditionally when it is safe to do so.
4276	*/
4277	static void hmR0SvmPostRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4278	{
4279	RT_NOREF(pVM);
4280	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4281
4282	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4283	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4284
4285	/* TSC read must be done early for maximum accuracy. */
4286	PSVMVMCB pVmcbNstGst = pMixedCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4287	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4288	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
4289	if (!(pVmcbNstGstCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4290	{
4291	/*
4292	* Undo what we did in hmR0SvmUpdateTscOffsettingNested() but don't restore the
4293	* nested-guest VMCB TSC offset here. It shall eventually be restored on #VMEXIT
4294	* later by HMSvmNstGstVmExitNotify().
4295	*/
4296	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcbNstGstCtrl->u64TSCOffset - pVmcbNstGstCache->u64TSCOffset);
4297	}
4298
4299	if (pSvmTransient->fRestoreTscAuxMsr)
4300	{
4301	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4302	CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4303	if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4304	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4305	}
4306
4307	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4308	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4309	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4310
4311	Assert(!(ASMGetFlags() & X86_EFL_IF));
4312	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4313	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4314
4315	/* Mark the VMCB-state cache as unmodified by VMM. */
4316	pVmcbNstGstCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL;
4317
4318	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4319	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4320	{
4321	Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4322	return;
4323	}
4324
4325	pSvmTransient->u64ExitCode = pVmcbNstGstCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4326	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcbNstGstCtrl->u64ExitCode);/* Update the #VMEXIT history array. */
4327	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4328	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4329
4330	Assert(!pVCpu->hm.s.svm.fSyncVTpr);
4331	hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcbNstGst); /* Save the nested-guest state from the VMCB to the
4332	guest-CPU context. */
4333	}
4334	#endif
4335
4336	/**
4337	* Performs some essential restoration of state after running guest code in
4338	* AMD-V.
4339	*
4340	* @param pVM The cross context VM structure.
4341	* @param pVCpu The cross context virtual CPU structure.
4342	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4343	* out-of-sync. Make sure to update the required fields
4344	* before using them.
4345	* @param pSvmTransient Pointer to the SVM transient structure.
4346	* @param rcVMRun Return code of VMRUN.
4347	*
4348	* @remarks Called with interrupts disabled.
4349	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4350	* unconditionally when it is safe to do so.
4351	*/
4352	static void hmR0SvmPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4353	{
4354	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4355
4356	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4357	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4358
4359	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4360	pVmcb->ctrl.u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4361
4362	/* TSC read must be done early for maximum accuracy. */
4363	if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4364	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcb->ctrl.u64TSCOffset);
4365
4366	if (pSvmTransient->fRestoreTscAuxMsr)
4367	{
4368	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4369	CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4370	if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4371	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4372	}
4373
4374	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4375	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4376	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4377
4378	Assert(!(ASMGetFlags() & X86_EFL_IF));
4379	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4380	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4381
4382	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4383	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4384	{
4385	Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4386	return;
4387	}
4388
4389	pSvmTransient->u64ExitCode = pVmcb->ctrl.u64ExitCode; /* Save the #VMEXIT reason. */
4390	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcb->ctrl.u64ExitCode); /* Update the #VMEXIT history array. */
4391	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4392	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4393
4394	hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcb); /* Save the guest state from the VMCB to the guest-CPU context. */
4395
4396	if (RT_LIKELY(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID))
4397	{
4398	if (pVCpu->hm.s.svm.fSyncVTpr)
4399	{
4400	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4401	if ( pVM->hm.s.fTPRPatchingActive
4402	&& (pMixedCtx->msrLSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4403	{
4404	int rc = APICSetTpr(pVCpu, pMixedCtx->msrLSTAR & 0xff);
4405	AssertRC(rc);
4406	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4407	}
4408	else if (pSvmTransient->u8GuestTpr != pVmcb->ctrl.IntCtrl.n.u8VTPR)
4409	{
4410	int rc = APICSetTpr(pVCpu, pVmcb->ctrl.IntCtrl.n.u8VTPR << 4);
4411	AssertRC(rc);
4412	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4413	}
4414	}
4415	}
4416	}
4417
4418
4419	/**
4420	* Runs the guest code using AMD-V.
4421	*
4422	* @returns VBox status code.
4423	* @param pVM The cross context VM structure.
4424	* @param pVCpu The cross context virtual CPU structure.
4425	* @param pCtx Pointer to the guest-CPU context.
4426	* @param pcLoops Pointer to the number of executed loops.
4427	*/
4428	static int hmR0SvmRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4429	{
4430	uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4431	Assert(pcLoops);
4432	Assert(*pcLoops <= cMaxResumeLoops);
4433
4434	SVMTRANSIENT SvmTransient;
4435	SvmTransient.fUpdateTscOffsetting = true;
4436
4437	int rc = VERR_INTERNAL_ERROR_5;
4438	for (;;)
4439	{
4440	Assert(!HMR0SuspendPending());
4441	HMSVM_ASSERT_CPU_SAFE();
4442
4443	/* Preparatory work for running guest code, this may force us to return
4444	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4445	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4446	rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4447	if (rc != VINF_SUCCESS)
4448	break;
4449
4450	/*
4451	* No longjmps to ring-3 from this point on!!!
4452	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4453	* This also disables flushing of the R0-logger instance (if any).
4454	*/
4455	hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
4456	rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4457
4458	/* Restore any residual host-state and save any bits shared between host
4459	and guest into the guest-CPU state. Re-enables interrupts! */
4460	hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
4461
4462	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4463	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4464	{
4465	if (rc == VINF_SUCCESS)
4466	rc = VERR_SVM_INVALID_GUEST_STATE;
4467	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4468	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4469	break;
4470	}
4471
4472	/* Handle the #VMEXIT. */
4473	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4474	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4475	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4476	rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4477	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4478	if (rc != VINF_SUCCESS)
4479	break;
4480	if (++(*pcLoops) >= cMaxResumeLoops)
4481	{
4482	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4483	rc = VINF_EM_RAW_INTERRUPT;
4484	break;
4485	}
4486	}
4487
4488	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4489	return rc;
4490	}
4491
4492
4493	/**
4494	* Runs the guest code using AMD-V in single step mode.
4495	*
4496	* @returns VBox status code.
4497	* @param pVM The cross context VM structure.
4498	* @param pVCpu The cross context virtual CPU structure.
4499	* @param pCtx Pointer to the guest-CPU context.
4500	* @param pcLoops Pointer to the number of executed loops.
4501	*/
4502	static int hmR0SvmRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4503	{
4504	uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4505	Assert(pcLoops);
4506	Assert(*pcLoops <= cMaxResumeLoops);
4507
4508	SVMTRANSIENT SvmTransient;
4509	SvmTransient.fUpdateTscOffsetting = true;
4510
4511	uint16_t uCsStart = pCtx->cs.Sel;
4512	uint64_t uRipStart = pCtx->rip;
4513
4514	int rc = VERR_INTERNAL_ERROR_5;
4515	for (;;)
4516	{
4517	Assert(!HMR0SuspendPending());
4518	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
4519	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
4520	(unsigned)RTMpCpuId(), *pcLoops));
4521
4522	/* Preparatory work for running guest code, this may force us to return
4523	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4524	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4525	rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4526	if (rc != VINF_SUCCESS)
4527	break;
4528
4529	/*
4530	* No longjmps to ring-3 from this point on!!!
4531	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4532	* This also disables flushing of the R0-logger instance (if any).
4533	*/
4534	VMMRZCallRing3Disable(pVCpu);
4535	VMMRZCallRing3RemoveNotification(pVCpu);
4536	hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
4537
4538	rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4539
4540	/*
4541	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
4542	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
4543	*/
4544	hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
4545	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4546	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4547	{
4548	if (rc == VINF_SUCCESS)
4549	rc = VERR_SVM_INVALID_GUEST_STATE;
4550	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4551	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4552	return rc;
4553	}
4554
4555	/* Handle the #VMEXIT. */
4556	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4557	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4558	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4559	rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4560	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4561	if (rc != VINF_SUCCESS)
4562	break;
4563	if (++(*pcLoops) >= cMaxResumeLoops)
4564	{
4565	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4566	rc = VINF_EM_RAW_INTERRUPT;
4567	break;
4568	}
4569
4570	/*
4571	* Did the RIP change, if so, consider it a single step.
4572	* Otherwise, make sure one of the TFs gets set.
4573	*/
4574	if ( pCtx->rip != uRipStart
4575	\|\| pCtx->cs.Sel != uCsStart)
4576	{
4577	rc = VINF_EM_DBG_STEPPED;
4578	break;
4579	}
4580	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
4581	}
4582
4583	/*
4584	* Clear the X86_EFL_TF if necessary.
4585	*/
4586	if (pVCpu->hm.s.fClearTrapFlag)
4587	{
4588	pVCpu->hm.s.fClearTrapFlag = false;
4589	pCtx->eflags.Bits.u1TF = 0;
4590	}
4591
4592	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4593	return rc;
4594	}
4595
4596	#ifdef VBOX_WITH_NESTED_HWVIRT
4597	/**
4598	* Runs the nested-guest code using AMD-V.
4599	*
4600	* @returns VBox status code.
4601	* @param pVM The cross context VM structure.
4602	* @param pVCpu The cross context virtual CPU structure.
4603	* @param pCtx Pointer to the guest-CPU context.
4604	* @param pcLoops Pointer to the number of executed loops. If we're switching
4605	* from the guest-code execution loop to this nested-guest
4606	* execution loop pass the remainder value, else pass 0.
4607	*/
4608	static int hmR0SvmRunGuestCodeNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4609	{
4610	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4611	Assert(pcLoops);
4612	Assert(*pcLoops <= pVM->hm.s.cMaxResumeLoops);
4613
4614	SVMTRANSIENT SvmTransient;
4615	SvmTransient.fUpdateTscOffsetting = true;
4616
4617	int rc = VERR_INTERNAL_ERROR_4;
4618	for (;;)
4619	{
4620	Assert(!HMR0SuspendPending());
4621	HMSVM_ASSERT_CPU_SAFE();
4622
4623	/* Preparatory work for running nested-guest code, this may force us to return
4624	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4625	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4626	rc = hmR0SvmPreRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient);
4627	if (rc != VINF_SUCCESS)
4628	break;
4629
4630	/*
4631	* No longjmps to ring-3 from this point on!!!
4632	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4633	* This also disables flushing of the R0-logger instance (if any).
4634	*/
4635	hmR0SvmPreRunGuestCommittedNested(pVM, pVCpu, pCtx, &SvmTransient);
4636
4637	rc = hmR0SvmRunGuestNested(pVM, pVCpu, pCtx);
4638
4639	/* Restore any residual host-state and save any bits shared between host
4640	and guest into the guest-CPU state. Re-enables interrupts! */
4641	hmR0SvmPostRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient, rc);
4642
4643	/** @todo This needs some work... we probably should cause a \#VMEXIT on
4644	* SVM_EXIT_INVALID and handle rc != VINF_SUCCESS differently. */
4645	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4646	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4647	{
4648	if (rc == VINF_SUCCESS)
4649	rc = VERR_SVM_INVALID_GUEST_STATE;
4650	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4651	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4652	break;
4653	}
4654
4655	/* Handle the #VMEXIT. */
4656	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4657	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4658	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
4659	rc = hmR0SvmHandleExitNested(pVCpu, pCtx, &SvmTransient);
4660	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4661	if (rc != VINF_SUCCESS)
4662	break;
4663	if (++(*pcLoops) >= pVM->hm.s.cMaxResumeLoops)
4664	{
4665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4666	rc = VINF_EM_RAW_INTERRUPT;
4667	break;
4668	}
4669
4670	/** @todo handle single-stepping */
4671	}
4672
4673	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4674	return rc;
4675	}
4676	#endif
4677
4678
4679	/**
4680	* Runs the guest code using AMD-V.
4681	*
4682	* @returns Strict VBox status code.
4683	* @param pVM The cross context VM structure.
4684	* @param pVCpu The cross context virtual CPU structure.
4685	* @param pCtx Pointer to the guest-CPU context.
4686	*/
4687	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4688	{
4689	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4690	HMSVM_ASSERT_PREEMPT_SAFE();
4691	VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, pCtx);
4692
4693	uint32_t cLoops = 0;
4694	int rc;
4695	#ifdef VBOX_WITH_NESTED_HWVIRT
4696	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4697	#endif
4698	{
4699	if (!pVCpu->hm.s.fSingleInstruction)
4700	rc = hmR0SvmRunGuestCodeNormal(pVM, pVCpu, pCtx, &cLoops);
4701	else
4702	rc = hmR0SvmRunGuestCodeStep(pVM, pVCpu, pCtx, &cLoops);
4703	}
4704	#ifdef VBOX_WITH_NESTED_HWVIRT
4705	else
4706	{
4707	rc = VINF_SVM_VMRUN;
4708	}
4709
4710	/* Re-check the nested-guest condition here as we may be transitioning from the normal
4711	execution loop into the nested-guest, hence this is not placed in the 'else' part above. */
4712	if (rc == VINF_SVM_VMRUN)
4713	{
4714	rc = hmR0SvmRunGuestCodeNested(pVM, pVCpu, pCtx, &cLoops);
4715	if (rc == VINF_SVM_VMEXIT)
4716	rc = VINF_SUCCESS;
4717	}
4718	#endif
4719
4720	/* Fixup error codes. */
4721	if (rc == VERR_EM_INTERPRETER)
4722	rc = VINF_EM_RAW_EMULATE_INSTR;
4723	else if (rc == VINF_EM_RESET)
4724	rc = VINF_EM_TRIPLE_FAULT;
4725
4726	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4727	rc = hmR0SvmExitToRing3(pVM, pVCpu, pCtx, rc);
4728	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
4729	return rc;
4730	}
4731
4732
4733	#ifdef VBOX_WITH_NESTED_HWVIRT
4734	/**
4735	* Determines whether an IOIO intercept is active for the nested-guest or not.
4736	*
4737	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4738	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4739	*/
4740	static bool hmR0SvmIsIoInterceptActive(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4741	{
4742	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4743	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4744	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4745	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4746	const uint8_t iEffSeg = pIoExitInfo->n.u3SEG;
4747	const bool fRep = pIoExitInfo->n.u1REP;
4748	const bool fStrIo = pIoExitInfo->n.u1STR;
4749
4750	return HMSvmIsIOInterceptActive(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4751	NULL /* pIoExitInfo */);
4752	}
4753
4754
4755	/**
4756	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4757	* SVM_EXIT_INVALID).
4758	*
4759	* @returns VBox status code (informational status codes included).
4760	* @param pVCpu The cross context virtual CPU structure.
4761	* @param pCtx Pointer to the guest-CPU context.
4762	* @param pSvmTransient Pointer to the SVM transient structure.
4763	*/
4764	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4765	{
4766	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4767	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4768	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4769
4770	#define HM_SVM_VMEXIT_NESTED(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4771	VBOXSTRICTRC_TODO(IEMExecSvmVmexit(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2))
4772
4773	/*
4774	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected
4775	* by the nested-guest. If it isn't, it should be handled by the (outer) guest.
4776	*/
4777	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4778	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4779	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4780	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4781	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4782
4783	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4784	switch (uExitCode)
4785	{
4786	case SVM_EXIT_CPUID:
4787	{
4788	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4789	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4790	return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
4791	}
4792
4793	case SVM_EXIT_RDTSC:
4794	{
4795	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4796	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4797	return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
4798	}
4799
4800	case SVM_EXIT_RDTSCP:
4801	{
4802	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4803	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4804	return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
4805	}
4806
4807
4808	case SVM_EXIT_MONITOR:
4809	{
4810	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4811	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4812	return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
4813	}
4814
4815	case SVM_EXIT_MWAIT:
4816	{
4817	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4818	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4819	return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
4820	}
4821
4822	case SVM_EXIT_HLT:
4823	{
4824	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4825	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4826	return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
4827	}
4828
4829	case SVM_EXIT_MSR:
4830	{
4831	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4832	{
4833	uint32_t const idMsr = pCtx->ecx;
4834	uint16_t offMsrpm;
4835	uint32_t uMsrpmBit;
4836	int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4837	if (RT_SUCCESS(rc))
4838	{
4839	void const *pvMsrBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4840	bool const fInterceptRead = ASMBitTest(pvMsrBitmap, (offMsrpm << 3) + uMsrpmBit);
4841	bool const fInterceptWrite = ASMBitTest(pvMsrBitmap, (offMsrpm << 3) + uMsrpmBit + 1);
4842
4843	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4844	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4845	{
4846	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4847	}
4848	}
4849	else
4850	{
4851	/*
4852	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4853	* See AMD-V spec. "15.11 MSR Intercepts".
4854	*/
4855	Assert(rc == VERR_OUT_OF_RANGE);
4856	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4857	}
4858	}
4859	return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
4860	}
4861
4862	case SVM_EXIT_IOIO:
4863	{
4864	/*
4865	* Figure out if the IO port access is intercepted by the nested-guest.
4866	*/
4867	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4868	{
4869	void *pvIoBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvIoBitmap);
4870	SVMIOIOEXITINFO IoExitInfo;
4871	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4872	bool const fIntercept = hmR0SvmIsIoInterceptActive(pvIoBitmap, &IoExitInfo);
4873	if (fIntercept)
4874	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4875	}
4876	return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
4877	}
4878
4879	case SVM_EXIT_EXCEPTION_14: /* X86_XCPT_PF */
4880	{
4881	PVM pVM = pVCpu->CTX_SUFF(pVM);
4882	if (pVM->hm.s.fNestedPaging)
4883	{
4884	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4885	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4886
4887	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4888	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4889	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4890
4891	/* If the nested-guest is not intercepting #PFs, forward the #PF to the nested-guest. */
4892	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
4893	return VINF_SUCCESS;
4894	}
4895	return hmR0SvmExitXcptPFNested(pVCpu, pCtx,pSvmTransient);
4896	}
4897
4898	case SVM_EXIT_EXCEPTION_7: /* X86_XCPT_NM */
4899	{
4900	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_NM))
4901	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4902	hmR0SvmSetPendingXcptNM(pVCpu);
4903	return VINF_SUCCESS;
4904	}
4905
4906	case SVM_EXIT_EXCEPTION_6: /* X86_XCPT_UD */
4907	{
4908	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4909	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4910	hmR0SvmSetPendingXcptUD(pVCpu);
4911	return VINF_SUCCESS;
4912	}
4913
4914	case SVM_EXIT_EXCEPTION_16: /* X86_XCPT_MF */
4915	{
4916	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4917	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4918	hmR0SvmSetPendingXcptMF(pVCpu);
4919	return VINF_SUCCESS;
4920	}
4921
4922	case SVM_EXIT_EXCEPTION_1: /* X86_XCPT_DB */
4923	{
4924	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4925	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4926	return hmR0SvmNestedExitXcptDB(pVCpu, pCtx, pSvmTransient);
4927	}
4928
4929	case SVM_EXIT_EXCEPTION_17: /* X86_XCPT_AC */
4930	{
4931	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4932	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4933	return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
4934	}
4935
4936	case SVM_EXIT_EXCEPTION_3: /* X86_XCPT_BP */
4937	{
4938	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
4939	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4940	return hmR0SvmNestedExitXcptBP(pVCpu, pCtx, pSvmTransient);
4941	}
4942
4943	case SVM_EXIT_READ_CR0:
4944	case SVM_EXIT_READ_CR3:
4945	case SVM_EXIT_READ_CR4:
4946	{
4947	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
4948	if (HMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
4949	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4950	return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
4951	}
4952
4953	case SVM_EXIT_WRITE_CR0:
4954	case SVM_EXIT_WRITE_CR3:
4955	case SVM_EXIT_WRITE_CR4:
4956	case SVM_EXIT_WRITE_CR8: /** @todo Shouldn't writes to CR8 go to V_TPR instead since we run with V_INTR_MASKING set?? */
4957	{
4958	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
4959	Log4(("hmR0SvmHandleExitNested: Write CRx: u16InterceptWrCRx=%#x u64ExitCode=%#RX64 %#x\n",
4960	pVmcbNstGstCtrl->u16InterceptWrCRx, pSvmTransient->u64ExitCode, uCr));
4961
4962	if (HMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
4963	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4964	return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
4965	}
4966
4967	case SVM_EXIT_PAUSE:
4968	{
4969	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
4970	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4971	return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
4972	}
4973
4974	case SVM_EXIT_VINTR:
4975	{
4976	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
4977	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4978	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
4979	}
4980
4981	case SVM_EXIT_INTR:
4982	case SVM_EXIT_NMI:
4983	case SVM_EXIT_SMI:
4984	{
4985	/*
4986	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
4987	*
4988	* Although we don't intercept SMIs, the nested-guest might. Therefore, we
4989	* might get an SMI #VMEXIT here so simply ignore rather than causing a
4990	* corresponding nested-guest #VMEXIT.
4991	*/
4992	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
4993	}
4994
4995	case SVM_EXIT_FERR_FREEZE:
4996	{
4997	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
4998	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4999	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5000	}
5001
5002	case SVM_EXIT_INVLPG:
5003	{
5004	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5005	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5006	return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5007	}
5008
5009	case SVM_EXIT_WBINVD:
5010	{
5011	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5012	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5013	return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5014	}
5015
5016	case SVM_EXIT_INVD:
5017	{
5018	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5019	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5020	return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5021	}
5022
5023	case SVM_EXIT_RDPMC:
5024	{
5025	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5026	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5027	return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5028	}
5029
5030	default:
5031	{
5032	switch (uExitCode)
5033	{
5034	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5035	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5036	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5037	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5038	{
5039	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5040	if (HMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5041	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5042	return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5043	}
5044
5045	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5046	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5047	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5048	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5049	{
5050	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5051	if (HMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5052	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5053	return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5054	}
5055
5056	/* The exceptions not handled here are already handled individually above (as they occur more frequently). */
5057	case SVM_EXIT_EXCEPTION_0: /case SVM_EXIT_EXCEPTION_1:/ case SVM_EXIT_EXCEPTION_2:
5058	/case SVM_EXIT_EXCEPTION_3:/ case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5:
5059	/case SVM_EXIT_EXCEPTION_6:/ /case SVM_EXIT_EXCEPTION_7:/ case SVM_EXIT_EXCEPTION_8:
5060	case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11:
5061	case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13: /case SVM_EXIT_EXCEPTION_14:/
5062	case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: /case SVM_EXIT_EXCEPTION_17:/
5063	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_20:
5064	case SVM_EXIT_EXCEPTION_21: case SVM_EXIT_EXCEPTION_22: case SVM_EXIT_EXCEPTION_23:
5065	case SVM_EXIT_EXCEPTION_24: case SVM_EXIT_EXCEPTION_25: case SVM_EXIT_EXCEPTION_26:
5066	case SVM_EXIT_EXCEPTION_27: case SVM_EXIT_EXCEPTION_28: case SVM_EXIT_EXCEPTION_29:
5067	case SVM_EXIT_EXCEPTION_30: case SVM_EXIT_EXCEPTION_31:
5068	{
5069	uint8_t const uVector = uExitCode - SVM_EXIT_EXCEPTION_0;
5070	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5071	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5072	return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5073	}
5074
5075	case SVM_EXIT_XSETBV:
5076	{
5077	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5078	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5079	return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5080	}
5081
5082	case SVM_EXIT_TASK_SWITCH:
5083	{
5084	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5085	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5086	return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5087	}
5088
5089	case SVM_EXIT_IRET:
5090	{
5091	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5092	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5093	return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5094	}
5095
5096	case SVM_EXIT_SHUTDOWN:
5097	{
5098	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5099	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5100	return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5101	}
5102
5103	case SVM_EXIT_VMMCALL:
5104	{
5105	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5106	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5107	return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5108	}
5109
5110	case SVM_EXIT_CLGI:
5111	{
5112	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5113	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5114	return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5115	}
5116
5117	case SVM_EXIT_STGI:
5118	{
5119	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5120	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5121	return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5122	}
5123
5124	case SVM_EXIT_VMLOAD:
5125	{
5126	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5127	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5128	return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5129	}
5130
5131	case SVM_EXIT_VMSAVE:
5132	{
5133	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5134	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5135	return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5136	}
5137
5138	case SVM_EXIT_INVLPGA:
5139	{
5140	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5141	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5142	return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5143	}
5144
5145	case SVM_EXIT_VMRUN:
5146	{
5147	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5148	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5149	return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5150	}
5151
5152	case SVM_EXIT_RSM:
5153	{
5154	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5155	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5156	hmR0SvmSetPendingXcptUD(pVCpu);
5157	return VINF_SUCCESS;
5158	}
5159
5160	case SVM_EXIT_SKINIT:
5161	{
5162	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5163	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5164	hmR0SvmSetPendingXcptUD(pVCpu);
5165	return VINF_SUCCESS;
5166	}
5167
5168	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5169	case SVM_EXIT_NPF: /* We don't yet support nested-paging for nested-guests, so this should never happen. */
5170	{
5171	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5172	}
5173
5174	default:
5175	{
5176	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5177	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5178	return VERR_SVM_UNKNOWN_EXIT;
5179	}
5180	}
5181	}
5182	}
5183	/* not reached */
5184
5185	#undef HM_SVM_VMEXIT_NESTED
5186	}
5187	#endif
5188
5189
5190	/**
5191	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5192	*
5193	* @returns VBox status code (informational status codes included).
5194	* @param pVCpu The cross context virtual CPU structure.
5195	* @param pCtx Pointer to the guest-CPU context.
5196	* @param pSvmTransient Pointer to the SVM transient structure.
5197	*/
5198	static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5199	{
5200	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5201	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5202
5203	/*
5204	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs for most guests under
5205	* normal workloads (for some definition of "normal").
5206	*/
5207	uint32_t u32ExitCode = pSvmTransient->u64ExitCode;
5208	switch (pSvmTransient->u64ExitCode)
5209	{
5210	case SVM_EXIT_NPF:
5211	return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
5212
5213	case SVM_EXIT_IOIO:
5214	return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
5215
5216	case SVM_EXIT_RDTSC:
5217	return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
5218
5219	case SVM_EXIT_RDTSCP:
5220	return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
5221
5222	case SVM_EXIT_CPUID:
5223	return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
5224
5225	case SVM_EXIT_EXCEPTION_14: /* X86_XCPT_PF */
5226	return hmR0SvmExitXcptPF(pVCpu, pCtx, pSvmTransient);
5227
5228	case SVM_EXIT_EXCEPTION_7: /* X86_XCPT_NM */
5229	return hmR0SvmExitXcptNM(pVCpu, pCtx, pSvmTransient);
5230
5231	case SVM_EXIT_EXCEPTION_6: /* X86_XCPT_UD */
5232	return hmR0SvmExitXcptUD(pVCpu, pCtx, pSvmTransient);
5233
5234	case SVM_EXIT_EXCEPTION_16: /* X86_XCPT_MF */
5235	return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
5236
5237	case SVM_EXIT_EXCEPTION_1: /* X86_XCPT_DB */
5238	return hmR0SvmExitXcptDB(pVCpu, pCtx, pSvmTransient);
5239
5240	case SVM_EXIT_EXCEPTION_17: /* X86_XCPT_AC */
5241	return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5242
5243	case SVM_EXIT_EXCEPTION_3: /* X86_XCPT_BP */
5244	return hmR0SvmExitXcptBP(pVCpu, pCtx, pSvmTransient);
5245
5246	case SVM_EXIT_MONITOR:
5247	return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
5248
5249	case SVM_EXIT_MWAIT:
5250	return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
5251
5252	case SVM_EXIT_HLT:
5253	return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
5254
5255	case SVM_EXIT_READ_CR0:
5256	case SVM_EXIT_READ_CR3:
5257	case SVM_EXIT_READ_CR4:
5258	return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5259
5260	case SVM_EXIT_WRITE_CR0:
5261	case SVM_EXIT_WRITE_CR3:
5262	case SVM_EXIT_WRITE_CR4:
5263	case SVM_EXIT_WRITE_CR8:
5264	return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5265
5266	case SVM_EXIT_PAUSE:
5267	return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5268
5269	case SVM_EXIT_VMMCALL:
5270	return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5271
5272	case SVM_EXIT_VINTR:
5273	return hmR0SvmExitVIntr(pVCpu, pCtx, pSvmTransient);
5274
5275	case SVM_EXIT_INTR:
5276	case SVM_EXIT_FERR_FREEZE:
5277	case SVM_EXIT_NMI:
5278	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5279
5280	case SVM_EXIT_MSR:
5281	return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
5282
5283	case SVM_EXIT_INVLPG:
5284	return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5285
5286	case SVM_EXIT_WBINVD:
5287	return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5288
5289	case SVM_EXIT_INVD:
5290	return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5291
5292	case SVM_EXIT_RDPMC:
5293	return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5294
5295	default:
5296	{
5297	switch (pSvmTransient->u64ExitCode)
5298	{
5299	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5300	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5301	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5302	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5303	return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5304
5305	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5306	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5307	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5308	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5309	return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5310
5311	case SVM_EXIT_XSETBV:
5312	return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5313
5314	case SVM_EXIT_TASK_SWITCH:
5315	return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5316
5317	case SVM_EXIT_IRET:
5318	return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5319
5320	case SVM_EXIT_SHUTDOWN:
5321	return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5322
5323	case SVM_EXIT_SMI:
5324	case SVM_EXIT_INIT:
5325	{
5326	/*
5327	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5328	* If it ever does, we want to know about it so log the exit code and bail.
5329	*/
5330	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5331	}
5332
5333	#ifdef VBOX_WITH_NESTED_HWVIRT
5334	case SVM_EXIT_CLGI: return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5335	case SVM_EXIT_STGI: return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5336	case SVM_EXIT_VMLOAD: return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5337	case SVM_EXIT_VMSAVE: return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5338	case SVM_EXIT_INVLPGA: return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5339	case SVM_EXIT_VMRUN: return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5340	#else
5341	case SVM_EXIT_CLGI:
5342	case SVM_EXIT_STGI:
5343	case SVM_EXIT_VMLOAD:
5344	case SVM_EXIT_VMSAVE:
5345	case SVM_EXIT_INVLPGA:
5346	case SVM_EXIT_VMRUN:
5347	#endif
5348	case SVM_EXIT_RSM:
5349	case SVM_EXIT_SKINIT:
5350	{
5351	hmR0SvmSetPendingXcptUD(pVCpu);
5352	return VINF_SUCCESS;
5353	}
5354
5355	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5356	case SVM_EXIT_EXCEPTION_0: /* X86_XCPT_DE */
5357	/* SVM_EXIT_EXCEPTION_1: / / X86_XCPT_DB - Handled above. */
5358	case SVM_EXIT_EXCEPTION_2: /* X86_XCPT_NMI */
5359	/* SVM_EXIT_EXCEPTION_3: / / X86_XCPT_BP - Handled above. */
5360	case SVM_EXIT_EXCEPTION_4: /* X86_XCPT_OF */
5361	case SVM_EXIT_EXCEPTION_5: /* X86_XCPT_BR */
5362	/* SVM_EXIT_EXCEPTION_6: / / X86_XCPT_UD - Handled above. */
5363	/* SVM_EXIT_EXCEPTION_7: / / X86_XCPT_NM - Handled above. */
5364	case SVM_EXIT_EXCEPTION_8: /* X86_XCPT_DF */
5365	case SVM_EXIT_EXCEPTION_9: /* X86_XCPT_CO_SEG_OVERRUN */
5366	case SVM_EXIT_EXCEPTION_10: /* X86_XCPT_TS */
5367	case SVM_EXIT_EXCEPTION_11: /* X86_XCPT_NP */
5368	case SVM_EXIT_EXCEPTION_12: /* X86_XCPT_SS */
5369	case SVM_EXIT_EXCEPTION_13: /* X86_XCPT_GP */
5370	/* SVM_EXIT_EXCEPTION_14: / / X86_XCPT_PF - Handled above. */
5371	case SVM_EXIT_EXCEPTION_15: /* Reserved. */
5372	/* SVM_EXIT_EXCEPTION_16: / / X86_XCPT_MF - Handled above. */
5373	/* SVM_EXIT_EXCEPTION_17: / / X86_XCPT_AC - Handled above. */
5374	case SVM_EXIT_EXCEPTION_18: /* X86_XCPT_MC */
5375	case SVM_EXIT_EXCEPTION_19: /* X86_XCPT_XF */
5376	case SVM_EXIT_EXCEPTION_20: case SVM_EXIT_EXCEPTION_21: case SVM_EXIT_EXCEPTION_22:
5377	case SVM_EXIT_EXCEPTION_23: case SVM_EXIT_EXCEPTION_24: case SVM_EXIT_EXCEPTION_25:
5378	case SVM_EXIT_EXCEPTION_26: case SVM_EXIT_EXCEPTION_27: case SVM_EXIT_EXCEPTION_28:
5379	case SVM_EXIT_EXCEPTION_29: case SVM_EXIT_EXCEPTION_30: case SVM_EXIT_EXCEPTION_31:
5380	{
5381	/** @todo r=ramshankar; We should be doing
5382	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY here! */
5383	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
5384	SVMEVENT Event;
5385	Event.u = 0;
5386	Event.n.u1Valid = 1;
5387	Event.n.u3Type = SVM_EVENT_EXCEPTION;
5388	Event.n.u8Vector = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0;
5389
5390	switch (Event.n.u8Vector)
5391	{
5392	case X86_XCPT_DE:
5393	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
5394	break;
5395
5396	case X86_XCPT_NP:
5397	Event.n.u1ErrorCodeValid = 1;
5398	Event.n.u32ErrorCode = pVmcb->ctrl.u64ExitInfo1;
5399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
5400	break;
5401
5402	case X86_XCPT_SS:
5403	Event.n.u1ErrorCodeValid = 1;
5404	Event.n.u32ErrorCode = pVmcb->ctrl.u64ExitInfo1;
5405	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
5406	break;
5407
5408	case X86_XCPT_GP:
5409	Event.n.u1ErrorCodeValid = 1;
5410	Event.n.u32ErrorCode = pVmcb->ctrl.u64ExitInfo1;
5411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
5412	break;
5413
5414	default:
5415	AssertMsgFailed(("hmR0SvmHandleExit: Unexpected exit caused by exception %#x\n", Event.n.u8Vector));
5416	pVCpu->hm.s.u32HMError = Event.n.u8Vector;
5417	return VERR_SVM_UNEXPECTED_XCPT_EXIT;
5418	}
5419
5420	Log4(("#Xcpt: Vector=%#x at CS:RIP=%04x:%RGv\n", Event.n.u8Vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
5421	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
5422	return VINF_SUCCESS;
5423	}
5424	#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5425
5426	default:
5427	{
5428	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#x\n", u32ExitCode));
5429	pVCpu->hm.s.u32HMError = u32ExitCode;
5430	return VERR_SVM_UNKNOWN_EXIT;
5431	}
5432	}
5433	}
5434	}
5435	/* not reached */
5436	}
5437
5438
5439	#ifdef DEBUG
5440	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5441	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5442	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5443
5444	# define HMSVM_ASSERT_PREEMPT_CPUID() \
5445	do \
5446	{ \
5447	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5448	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5449	} while (0)
5450
5451	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() \
5452	do { \
5453	AssertPtr(pVCpu); \
5454	AssertPtr(pCtx); \
5455	AssertPtr(pSvmTransient); \
5456	Assert(ASMIntAreEnabled()); \
5457	HMSVM_ASSERT_PREEMPT_SAFE(); \
5458	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5459	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
5460	HMSVM_ASSERT_PREEMPT_SAFE(); \
5461	if (VMMR0IsLogFlushDisabled(pVCpu)) \
5462	HMSVM_ASSERT_PREEMPT_CPUID(); \
5463	} while (0)
5464	#else /* Release builds */
5465	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() do { NOREF(pVCpu); NOREF(pCtx); NOREF(pSvmTransient); } while (0)
5466	#endif
5467
5468
5469	/**
5470	* Worker for hmR0SvmInterpretInvlpg().
5471	*
5472	* @return VBox status code.
5473	* @param pVCpu The cross context virtual CPU structure.
5474	* @param pCpu Pointer to the disassembler state.
5475	* @param pCtx The guest CPU context.
5476	*/
5477	static int hmR0SvmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTX pCtx)
5478	{
5479	DISQPVPARAMVAL Param1;
5480	RTGCPTR GCPtrPage;
5481
5482	int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), pCpu, &pCpu->Param1, &Param1, DISQPVWHICH_SRC);
5483	if (RT_FAILURE(rc))
5484	return VERR_EM_INTERPRETER;
5485
5486	if ( Param1.type == DISQPV_TYPE_IMMEDIATE
5487	\|\| Param1.type == DISQPV_TYPE_ADDRESS)
5488	{
5489	if (!(Param1.flags & (DISQPV_FLAG_32 \| DISQPV_FLAG_64)))
5490	return VERR_EM_INTERPRETER;
5491
5492	GCPtrPage = Param1.val.val64;
5493	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), GCPtrPage);
5494	rc = VBOXSTRICTRC_VAL(rc2);
5495	}
5496	else
5497	{
5498	Log4(("hmR0SvmInterpretInvlPgEx invalid parameter type %#x\n", Param1.type));
5499	rc = VERR_EM_INTERPRETER;
5500	}
5501
5502	return rc;
5503	}
5504
5505
5506	/**
5507	* Interprets INVLPG.
5508	*
5509	* @returns VBox status code.
5510	* @retval VINF_* Scheduling instructions.
5511	* @retval VERR_EM_INTERPRETER Something we can't cope with.
5512	* @retval VERR_* Fatal errors.
5513	*
5514	* @param pVM The cross context VM structure.
5515	* @param pVCpu The cross context virtual CPU structure.
5516	* @param pCtx The guest CPU context.
5517	*
5518	* @remarks Updates the RIP if the instruction was executed successfully.
5519	*/
5520	static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5521	{
5522	/* Only allow 32 & 64 bit code. */
5523	if (CPUMGetGuestCodeBits(pVCpu) != 16)
5524	{
5525	PDISSTATE pDis = &pVCpu->hm.s.DisState;
5526	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
5527	if ( RT_SUCCESS(rc)
5528	&& pDis->pCurInstr->uOpcode == OP_INVLPG)
5529	{
5530	rc = hmR0SvmInterpretInvlPgEx(pVCpu, pDis, pCtx);
5531	if (RT_SUCCESS(rc))
5532	pCtx->rip += pDis->cbInstr;
5533	return rc;
5534	}
5535	else
5536	Log4(("hmR0SvmInterpretInvlpg: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
5537	}
5538	return VERR_EM_INTERPRETER;
5539	}
5540
5541
5542	#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5543	/**
5544	* Gets the IEM exception flags for the specified SVM event.
5545	*
5546	* @returns The IEM exception flags.
5547	* @param pEvent Pointer to the SVM event.
5548	*
5549	* @remarks This function currently only constructs flags required for
5550	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5551	* and CR2 aspects of an exception are not included).
5552	*/
5553	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
5554	{
5555	uint8_t const uEventType = pEvent->n.u3Type;
5556	uint32_t fIemXcptFlags;
5557	switch (uEventType)
5558	{
5559	case SVM_EVENT_EXCEPTION:
5560	/*
5561	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
5562	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
5563	*/
5564	if (pEvent->n.u8Vector == X86_XCPT_BP)
5565	{
5566	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
5567	break;
5568	}
5569	if (pEvent->n.u8Vector == X86_XCPT_OF)
5570	{
5571	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
5572	break;
5573	}
5574	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
5575	RT_FALL_THRU();
5576	case SVM_EVENT_NMI:
5577	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5578	break;
5579
5580	case SVM_EVENT_EXTERNAL_IRQ:
5581	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5582	break;
5583
5584	case SVM_EVENT_SOFTWARE_INT:
5585	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5586	break;
5587
5588	default:
5589	fIemXcptFlags = 0;
5590	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
5591	break;
5592	}
5593	return fIemXcptFlags;
5594	}
5595
5596	#else
5597	/**
5598	* Determines if an exception is a contributory exception.
5599	*
5600	* Contributory exceptions are ones which can cause double-faults unless the
5601	* original exception was a benign exception. Page-fault is intentionally not
5602	* included here as it's a conditional contributory exception.
5603	*
5604	* @returns @c true if the exception is contributory, @c false otherwise.
5605	* @param uVector The exception vector.
5606	*/
5607	DECLINLINE(bool) hmR0SvmIsContributoryXcpt(const uint32_t uVector)
5608	{
5609	switch (uVector)
5610	{
5611	case X86_XCPT_GP:
5612	case X86_XCPT_SS:
5613	case X86_XCPT_NP:
5614	case X86_XCPT_TS:
5615	case X86_XCPT_DE:
5616	return true;
5617	default:
5618	break;
5619	}
5620	return false;
5621	}
5622	#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5623
5624
5625	/**
5626	* Handle a condition that occurred while delivering an event through the guest
5627	* IDT.
5628	*
5629	* @returns VBox status code (informational error codes included).
5630	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
5631	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5632	* continue execution of the guest which will delivery the \#DF.
5633	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5634	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5635	*
5636	* @param pVCpu The cross context virtual CPU structure.
5637	* @param pCtx Pointer to the guest-CPU context.
5638	* @param pSvmTransient Pointer to the SVM transient structure.
5639	*
5640	* @remarks No-long-jump zone!!!
5641	*/
5642	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5643	{
5644	int rc = VINF_SUCCESS;
5645	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5646
5647	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
5648	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
5649	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
5650
5651	/* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code". The EXITINTINFO (if valid) contains the prior exception (IDT vector)
5652	* that was trying to be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector). */
5653	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
5654	{
5655	#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5656	IEMXCPTRAISE enmRaise;
5657	IEMXCPTRAISEINFO fRaiseInfo;
5658	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31;
5659	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5660	if (fExitIsHwXcpt)
5661	{
5662	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0;
5663	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
5664	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5665	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5666	}
5667	else
5668	{
5669	/*
5670	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF) then we
5671	* end up here.
5672	*
5673	* If the event was:
5674	* - a software interrupt, we can re-execute the instruction which will regenerate
5675	* the event.
5676	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
5677	* - a hardware exception or external interrupt, we re-inject it.
5678	*/
5679	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5680	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
5681	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5682	else
5683	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5684	}
5685
5686	switch (enmRaise)
5687	{
5688	case IEMXCPTRAISE_CURRENT_XCPT:
5689	case IEMXCPTRAISE_PREV_EVENT:
5690	{
5691	/* For software interrupts, we shall re-execute the instruction. */
5692	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
5693	{
5694	RTGCUINTPTR GCPtrFaultAddress = 0;
5695
5696	/* If we are re-injecting an NMI, clear NMI blocking. */
5697	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5698	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5699
5700	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
5701	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5702	pSvmTransient->fVectoringPF = true;
5703	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
5704	&& uIdtVector == X86_XCPT_PF)
5705	{
5706	/*
5707	* If the previous exception was a #PF, we need to recover the CR2 value.
5708	* This can't happen with shadow paging.
5709	*/
5710	GCPtrFaultAddress = pCtx->cr2;
5711	}
5712
5713	/*
5714	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
5715	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
5716	*/
5717	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5718	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5719	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
5720
5721	Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
5722	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
5723	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
5724	}
5725	break;
5726	}
5727
5728	case IEMXCPTRAISE_REEXEC_INSTR:
5729	{
5730	Assert(rc == VINF_SUCCESS);
5731	break;
5732	}
5733
5734	case IEMXCPTRAISE_DOUBLE_FAULT:
5735	{
5736	/*
5737	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5738	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5739	*/
5740	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5741	{
5742	pSvmTransient->fVectoringDoublePF = true;
5743	Assert(rc == VINF_SUCCESS);
5744	}
5745	else
5746	{
5747	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5748	hmR0SvmSetPendingXcptDF(pVCpu);
5749	rc = VINF_HM_DOUBLE_FAULT;
5750	}
5751	break;
5752	}
5753
5754	case IEMXCPTRAISE_TRIPLE_FAULT:
5755	{
5756	rc = VINF_EM_RESET;
5757	break;
5758	}
5759
5760	case IEMXCPTRAISE_CPU_HANG:
5761	{
5762	rc = VERR_EM_GUEST_CPU_HANG;
5763	break;
5764	}
5765
5766	default:
5767	{
5768	AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
5769	rc = VERR_SVM_IPE_2;
5770	break;
5771	}
5772	}
5773	#else
5774	uint8_t uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5775
5776	typedef enum
5777	{
5778	SVMREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5779	SVMREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5780	SVMREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5781	SVMREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5782	SVMREFLECTXCPT_NONE /* Nothing to reflect. */
5783	} SVMREFLECTXCPT;
5784
5785	SVMREFLECTXCPT enmReflect = SVMREFLECTXCPT_NONE;
5786	bool fReflectingNmi = false;
5787	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION)
5788	{
5789	if (pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31)
5790	{
5791	uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0);
5792
5793	#ifdef VBOX_STRICT
5794	if ( hmR0SvmIsContributoryXcpt(uIdtVector)
5795	&& uExitVector == X86_XCPT_PF)
5796	{
5797	Log4(("IDT: Contributory #PF idCpu=%u uCR2=%#RX64\n", pVCpu->idCpu, pCtx->cr2));
5798	}
5799	#endif
5800
5801	if ( uIdtVector == X86_XCPT_BP
5802	\|\| uIdtVector == X86_XCPT_OF)
5803	{
5804	/* Ignore INT3/INTO, just re-execute. See @bugref{8357}. */
5805	}
5806	else if ( uExitVector == X86_XCPT_PF
5807	&& uIdtVector == X86_XCPT_PF)
5808	{
5809	pSvmTransient->fVectoringDoublePF = true;
5810	Log4(("IDT: Vectoring double #PF uCR2=%#RX64\n", pCtx->cr2));
5811	}
5812	else if ( uExitVector == X86_XCPT_AC
5813	&& uIdtVector == X86_XCPT_AC)
5814	{
5815	enmReflect = SVMREFLECTXCPT_HANG;
5816	Log4(("IDT: Nested #AC - Bad guest\n"));
5817	}
5818	else if ( (pVmcb->ctrl.u32InterceptXcpt & HMSVM_CONTRIBUTORY_XCPT_MASK)
5819	&& hmR0SvmIsContributoryXcpt(uExitVector)
5820	&& ( hmR0SvmIsContributoryXcpt(uIdtVector)
5821	\|\| uIdtVector == X86_XCPT_PF))
5822	{
5823	enmReflect = SVMREFLECTXCPT_DF;
5824	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5825	uIdtVector, uExitVector));
5826	}
5827	else if (uIdtVector == X86_XCPT_DF)
5828	{
5829	enmReflect = SVMREFLECTXCPT_TF;
5830	Log4(("IDT: Pending vectoring triple-fault %#RX64 uIdtVector=%#x uExitVector=%#x\n",
5831	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5832	}
5833	else
5834	enmReflect = SVMREFLECTXCPT_XCPT;
5835	}
5836	else
5837	{
5838	/*
5839	* If event delivery caused an #VMEXIT that is not an exception (e.g. #NPF) then reflect the original
5840	* exception to the guest after handling the #VMEXIT.
5841	*/
5842	enmReflect = SVMREFLECTXCPT_XCPT;
5843	}
5844	}
5845	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXTERNAL_IRQ
5846	\|\| pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5847	{
5848	enmReflect = SVMREFLECTXCPT_XCPT;
5849	fReflectingNmi = RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI);
5850
5851	if (pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31)
5852	{
5853	uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0);
5854	if (uExitVector == X86_XCPT_PF)
5855	{
5856	pSvmTransient->fVectoringPF = true;
5857	Log4(("IDT: Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pCtx->cr2));
5858	}
5859	}
5860	}
5861	/* else: Ignore software interrupts (INT n) as they reoccur when restarting the instruction. */
5862
5863	switch (enmReflect)
5864	{
5865	case SVMREFLECTXCPT_XCPT:
5866	{
5867	/* If we are re-injecting the NMI, clear NMI blocking. */
5868	if (fReflectingNmi)
5869	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5870
5871	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5872	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5873	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, 0 /* GCPtrFaultAddress */);
5874
5875	/* If uExitVector is #PF, CR2 value will be updated from the VMCB if it's a guest #PF. See hmR0SvmExitXcptPF(). */
5876	Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32\n", pVmcb->ctrl.ExitIntInfo.u,
5877	!!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid, pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
5878	break;
5879	}
5880
5881	case SVMREFLECTXCPT_DF:
5882	{
5883	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5884	hmR0SvmSetPendingXcptDF(pVCpu);
5885	rc = VINF_HM_DOUBLE_FAULT;
5886	break;
5887	}
5888
5889	case SVMREFLECTXCPT_TF:
5890	{
5891	rc = VINF_EM_RESET;
5892	break;
5893	}
5894
5895	case SVMREFLECTXCPT_HANG:
5896	{
5897	rc = VERR_EM_GUEST_CPU_HANG;
5898	break;
5899	}
5900
5901	default:
5902	Assert(rc == VINF_SUCCESS);
5903	break;
5904	}
5905	#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5906	}
5907	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
5908	NOREF(pCtx);
5909	return rc;
5910	}
5911
5912
5913	/**
5914	* Advances the guest RIP making use of the CPU's NRIP_SAVE feature if
5915	* supported, otherwise advances the RIP by the number of bytes specified in
5916	* @a cb.
5917	*
5918	* @param pVCpu The cross context virtual CPU structure.
5919	* @param pCtx Pointer to the guest-CPU context.
5920	* @param cb RIP increment value in bytes.
5921	*
5922	* @remarks Use this function only from \#VMEXIT's where the NRIP value is valid
5923	* when NRIP_SAVE is supported by the CPU, otherwise use
5924	* hmR0SvmAdvanceRipDumb!
5925	*/
5926	DECLINLINE(void) hmR0SvmAdvanceRipHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
5927	{
5928	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
5929	if (fSupportsNextRipSave)
5930	{
5931	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5932	Assert(pVmcb->ctrl.u64NextRIP);
5933	AssertRelease(pVmcb->ctrl.u64NextRIP - pCtx->rip == cb); /* temporary, remove later */
5934	pCtx->rip = pVmcb->ctrl.u64NextRIP;
5935	}
5936	else
5937	pCtx->rip += cb;
5938
5939	HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
5940	}
5941
5942
5943	#ifdef VBOX_WITH_NESTED_HWVIRT
5944	/**
5945	* Gets the length of the current instruction if the CPU supports the NRIP_SAVE
5946	* feature. Otherwise, returns the value in @a cbLikely.
5947	*
5948	* @param pVCpu The cross context virtual CPU structure.
5949	* @param pCtx Pointer to the guest-CPU context.
5950	* @param cbLikely The likely instruction length.
5951	*/
5952	DECLINLINE(uint8_t) hmR0SvmGetInstrLengthHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint8_t cbLikely)
5953	{
5954	Assert(cbLikely <= 15); /* See Intel spec. 2.3.11 "AVX Instruction Length" */
5955	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
5956	if (fSupportsNextRipSave)
5957	{
5958	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5959	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
5960	Assert(cbInstr == cbLikely);
5961	return cbInstr;
5962	}
5963	return cbLikely;
5964	}
5965	#endif
5966
5967
5968	/**
5969	* Advances the guest RIP by the number of bytes specified in @a cb. This does
5970	* not make use of any hardware features to determine the instruction length.
5971	*
5972	* @param pVCpu The cross context virtual CPU structure.
5973	* @param pCtx Pointer to the guest-CPU context.
5974	* @param cb RIP increment value in bytes.
5975	*/
5976	DECLINLINE(void) hmR0SvmAdvanceRipDumb(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
5977	{
5978	pCtx->rip += cb;
5979	HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
5980	}
5981	#undef HMSVM_UPDATE_INTR_SHADOW
5982
5983
5984	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5985	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
5986	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5987
5988	/** @name \#VMEXIT handlers.
5989	* @{
5990	*/
5991
5992	/**
5993	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
5994	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
5995	*/
5996	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5997	{
5998	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
5999
6000	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6001	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6002	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6004
6005	/*
6006	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to signal -before- the timer
6007	* fires if the current interrupt is our own timer or a some other host interrupt. We also cannot examine what
6008	* interrupt it is until the host actually take the interrupt.
6009	*
6010	* Going back to executing guest code here unconditionally causes random scheduling problems (observed on an
6011	* AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6012	*/
6013	return VINF_EM_RAW_INTERRUPT;
6014	}
6015
6016
6017	/**
6018	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6019	*/
6020	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6021	{
6022	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6023
6024	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6025	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6026	int rc = VINF_SUCCESS;
6027	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6028	return rc;
6029	}
6030
6031
6032	/**
6033	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6034	*/
6035	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6036	{
6037	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6038
6039	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6040	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6041	int rc = VINF_SUCCESS;
6042	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6043	return rc;
6044	}
6045
6046
6047	/**
6048	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6049	*/
6050	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6051	{
6052	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6053	PVM pVM = pVCpu->CTX_SUFF(pVM);
6054	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6055	if (RT_LIKELY(rc == VINF_SUCCESS))
6056	{
6057	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6058	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6059	}
6060	else
6061	{
6062	AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6063	rc = VERR_EM_INTERPRETER;
6064	}
6065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6066	return rc;
6067	}
6068
6069
6070	/**
6071	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6072	*/
6073	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6074	{
6075	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6076	PVM pVM = pVCpu->CTX_SUFF(pVM);
6077	int rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6078	if (RT_LIKELY(rc == VINF_SUCCESS))
6079	{
6080	pSvmTransient->fUpdateTscOffsetting = true;
6081	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6082	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6083	}
6084	else
6085	{
6086	AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6087	rc = VERR_EM_INTERPRETER;
6088	}
6089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6090	return rc;
6091	}
6092
6093
6094	/**
6095	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6096	*/
6097	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6098	{
6099	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6100	int rc = EMInterpretRdtscp(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
6101	if (RT_LIKELY(rc == VINF_SUCCESS))
6102	{
6103	pSvmTransient->fUpdateTscOffsetting = true;
6104	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6105	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6106	}
6107	else
6108	{
6109	AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtscp failed with %Rrc\n", rc));
6110	rc = VERR_EM_INTERPRETER;
6111	}
6112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
6113	return rc;
6114	}
6115
6116
6117	/**
6118	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6119	*/
6120	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6121	{
6122	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6123	int rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6124	if (RT_LIKELY(rc == VINF_SUCCESS))
6125	{
6126	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6127	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6128	}
6129	else
6130	{
6131	AssertMsgFailed(("hmR0SvmExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6132	rc = VERR_EM_INTERPRETER;
6133	}
6134	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6135	return rc;
6136	}
6137
6138
6139	/**
6140	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
6141	*/
6142	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6143	{
6144	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6145	PVM pVM = pVCpu->CTX_SUFF(pVM);
6146	Assert(!pVM->hm.s.fNestedPaging);
6147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6148
6149	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6150	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6151	if ( fSupportsDecodeAssists
6152	&& fSupportsNextRipSave)
6153	{
6154	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6155	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6156	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
6157	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
6158	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6159	return VBOXSTRICTRC_VAL(rcStrict);
6160	}
6161
6162	int rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, pCtx); /* Updates RIP if successful. */
6163	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
6164	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6165	return rc;
6166	}
6167
6168
6169	/**
6170	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6171	*/
6172	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6173	{
6174	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6175
6176	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 1);
6177	int rc = EMShouldContinueAfterHalt(pVCpu, pCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6178	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
6180	if (rc != VINF_SUCCESS)
6181	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6182	return rc;
6183	}
6184
6185
6186	/**
6187	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6188	*/
6189	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6190	{
6191	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6192	int rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6193	if (RT_LIKELY(rc == VINF_SUCCESS))
6194	{
6195	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6196	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6197	}
6198	else
6199	{
6200	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6201	rc = VERR_EM_INTERPRETER;
6202	}
6203	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6204	return rc;
6205	}
6206
6207
6208	/**
6209	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6210	*/
6211	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6212	{
6213	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6214	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6215	int rc = VBOXSTRICTRC_VAL(rc2);
6216	if ( rc == VINF_EM_HALT
6217	\|\| rc == VINF_SUCCESS)
6218	{
6219	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6220
6221	if ( rc == VINF_EM_HALT
6222	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
6223	{
6224	rc = VINF_SUCCESS;
6225	}
6226	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6227	}
6228	else
6229	{
6230	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
6231	rc = VERR_EM_INTERPRETER;
6232	}
6233	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
6234	("hmR0SvmExitMwait: EMInterpretMWait failed rc=%Rrc\n", rc));
6235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
6236	return rc;
6237	}
6238
6239
6240	/**
6241	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6242	* \#VMEXIT.
6243	*/
6244	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6245	{
6246	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6247	return VINF_EM_RESET;
6248	}
6249
6250
6251	/**
6252	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6253	*/
6254	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6255	{
6256	RT_NOREF(pCtx);
6257	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64\n", pSvmTransient->u64ExitCode));
6258	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6259	return VERR_SVM_UNEXPECTED_EXIT;
6260	}
6261
6262
6263	/**
6264	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6265	*/
6266	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6267	{
6268	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6269
6270	Log4(("hmR0SvmExitReadCRx: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0]);
6272
6273	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6274	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6275	if ( fSupportsDecodeAssists
6276	&& fSupportsNextRipSave)
6277	{
6278	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6279	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6280	if (fMovCRx)
6281	{
6282	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6283	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6284	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6285	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6286	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6287	return VBOXSTRICTRC_VAL(rcStrict);
6288	}
6289	/* else: SMSW instruction, fall back below to IEM for this. */
6290	}
6291
6292	VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6293	int rc = VBOXSTRICTRC_VAL(rc2);
6294	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3,
6295	("hmR0SvmExitReadCRx: EMInterpretInstruction failed rc=%Rrc\n", rc));
6296	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6297	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6298	return rc;
6299	}
6300
6301
6302	/**
6303	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6304	*/
6305	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6306	{
6307	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6308
6309	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0;
6310	Assert(iCrReg <= 15);
6311
6312	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6313	bool fDecodedInstr = false;
6314	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6315	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6316	if ( fSupportsDecodeAssists
6317	&& fSupportsNextRipSave)
6318	{
6319	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6320	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6321	if (fMovCRx)
6322	{
6323	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6324	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6325	Log4(("hmR0SvmExitWriteCRx: Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6326	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6327	fDecodedInstr = true;
6328	}
6329	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6330	}
6331
6332	if (!fDecodedInstr)
6333	{
6334	Log4(("hmR0SvmExitWriteCRx: iCrReg=%#x\n", iCrReg));
6335	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(pCtx), NULL);
6336	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6337	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6338	rcStrict = VERR_EM_INTERPRETER;
6339	}
6340
6341	if (rcStrict == VINF_SUCCESS)
6342	{
6343	switch (iCrReg)
6344	{
6345	case 0: /* CR0. */
6346	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6347	break;
6348
6349	case 3: /* CR3. */
6350	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
6351	break;
6352
6353	case 4: /* CR4. */
6354	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
6355	break;
6356
6357	case 8: /* CR8 (TPR). */
6358	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6359	break;
6360
6361	default:
6362	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6363	pSvmTransient->u64ExitCode, iCrReg));
6364	break;
6365	}
6366	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6367	}
6368	else
6369	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_CHANGE_MODE \|\| rcStrict == VINF_PGM_SYNC_CR3);
6370	return VBOXSTRICTRC_TODO(rcStrict);
6371	}
6372
6373
6374	/**
6375	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6376	* \#VMEXIT.
6377	*/
6378	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6379	{
6380	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6381	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6382	PVM pVM = pVCpu->CTX_SUFF(pVM);
6383
6384	int rc;
6385	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6386	{
6387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6388	Log4(("MSR Write: idMsr=%#RX32\n", pCtx->ecx));
6389
6390	/* Handle TPR patching; intercepted LSTAR write. */
6391	if ( pVM->hm.s.fTPRPatchingActive
6392	&& pCtx->ecx == MSR_K8_LSTAR)
6393	{
6394	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6395	{
6396	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6397	int rc2 = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6398	AssertRC(rc2);
6399	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6400	}
6401	rc = VINF_SUCCESS;
6402	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6403	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6404	return rc;
6405	}
6406
6407	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6408	if (fSupportsNextRipSave)
6409	{
6410	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6411	if (RT_LIKELY(rc == VINF_SUCCESS))
6412	{
6413	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6414	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6415	}
6416	else
6417	AssertMsg( rc == VERR_EM_INTERPRETER
6418	\|\| rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: EMInterpretWrmsr failed rc=%Rrc\n", rc));
6419	}
6420	else
6421	{
6422	rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */));
6423	if (RT_LIKELY(rc == VINF_SUCCESS))
6424	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc); /* RIP updated by EMInterpretInstruction(). */
6425	else
6426	AssertMsg( rc == VERR_EM_INTERPRETER
6427	\|\| rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: WrMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6428	}
6429
6430	if (rc == VINF_SUCCESS)
6431	{
6432	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
6433	if ( pCtx->ecx >= MSR_IA32_X2APIC_START
6434	&& pCtx->ecx <= MSR_IA32_X2APIC_END)
6435	{
6436	/*
6437	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest(). When full APIC register
6438	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCB before
6439	* EMInterpretWrmsr() changes it.
6440	*/
6441	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6442	}
6443	else if (pCtx->ecx == MSR_K6_EFER)
6444	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
6445	else if (pCtx->ecx == MSR_IA32_TSC)
6446	pSvmTransient->fUpdateTscOffsetting = true;
6447	}
6448	}
6449	else
6450	{
6451	/* MSR Read access. */
6452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6453	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ);
6454	Log4(("MSR Read: idMsr=%#RX32\n", pCtx->ecx));
6455
6456	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6457	if (fSupportsNextRipSave)
6458	{
6459	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6460	if (RT_LIKELY(rc == VINF_SUCCESS))
6461	{
6462	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6463	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6464	}
6465	else
6466	AssertMsg( rc == VERR_EM_INTERPRETER
6467	\|\| rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: EMInterpretRdmsr failed rc=%Rrc\n", rc));
6468	}
6469	else
6470	{
6471	rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0));
6472	if (RT_UNLIKELY(rc != VINF_SUCCESS))
6473	{
6474	AssertMsg( rc == VERR_EM_INTERPRETER
6475	\|\| rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: RdMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6476	}
6477	/* RIP updated by EMInterpretInstruction(). */
6478	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6479	}
6480	}
6481
6482	/* RIP has been updated by EMInterpret[Rd\|Wr]msr() or EMInterpretInstruction(). */
6483	return rc;
6484	}
6485
6486
6487	/**
6488	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6489	*/
6490	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6491	{
6492	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6494
6495	/** @todo Stepping with nested-guest. */
6496	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6497	{
6498	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6499	if (pSvmTransient->fWasGuestDebugStateActive)
6500	{
6501	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6502	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6503	return VERR_SVM_UNEXPECTED_EXIT;
6504	}
6505
6506	/*
6507	* Lazy DR0-3 loading.
6508	*/
6509	if (!pSvmTransient->fWasHyperDebugStateActive)
6510	{
6511	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6512	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6513
6514	/* Don't intercept DRx read and writes. */
6515	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6516	pVmcb->ctrl.u16InterceptRdDRx = 0;
6517	pVmcb->ctrl.u16InterceptWrDRx = 0;
6518	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6519
6520	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6521	VMMRZCallRing3Disable(pVCpu);
6522	HM_DISABLE_PREEMPT();
6523
6524	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6525	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6526	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
6527
6528	HM_RESTORE_PREEMPT();
6529	VMMRZCallRing3Enable(pVCpu);
6530
6531	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6532	return VINF_SUCCESS;
6533	}
6534	}
6535
6536	/*
6537	* Interpret the read/writing of DRx.
6538	*/
6539	/** @todo Decode assist. */
6540	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6541	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6542	if (RT_LIKELY(rc == VINF_SUCCESS))
6543	{
6544	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6545	/** @todo CPUM should set this flag! */
6546	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6547	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6548	}
6549	else
6550	Assert(rc == VERR_EM_INTERPRETER);
6551	return VBOXSTRICTRC_TODO(rc);
6552	}
6553
6554
6555	/**
6556	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6557	*/
6558	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6559	{
6560	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6561	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6562	int rc = hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
6563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6564	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6565	return rc;
6566	}
6567
6568
6569	/**
6570	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6571	*/
6572	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6573	{
6574	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6575
6576	/** @todo decode assists... */
6577	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6578	if (rcStrict == VINF_IEM_RAISED_XCPT)
6579	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6580
6581	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6582	Log4(("hmR0SvmExitXsetbv: New XCR0=%#RX64 fLoadSaveGuestXcr0=%d (cr4=%RX64) rcStrict=%Rrc\n",
6583	pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0, pCtx->cr4, VBOXSTRICTRC_VAL(rcStrict)));
6584
6585	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6586	return VBOXSTRICTRC_TODO(rcStrict);
6587	}
6588
6589
6590	/**
6591	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6592	*/
6593	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6594	{
6595	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6596
6597	/* I/O operation lookup arrays. */
6598	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
6599	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
6600	the result (in AL/AX/EAX). */
6601	Log4(("hmR0SvmExitIOInstr: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6602
6603	PVM pVM = pVCpu->CTX_SUFF(pVM);
6604	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6605
6606	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
6607	SVMIOIOEXITINFO IoExitInfo;
6608	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
6609	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
6610	uint32_t cbValue = s_aIOSize[uIOWidth];
6611	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
6612
6613	if (RT_UNLIKELY(!cbValue))
6614	{
6615	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
6616	return VERR_EM_INTERPRETER;
6617	}
6618
6619	VBOXSTRICTRC rcStrict;
6620	bool fUpdateRipAlready = false;
6621	if (IoExitInfo.n.u1STR)
6622	{
6623	#ifdef VBOX_WITH_2ND_IEM_STEP
6624	/* INS/OUTS - I/O String instruction. */
6625	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
6626	* in EXITINFO1? Investigate once this thing is up and running. */
6627	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
6628	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
6629	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
6630	static IEMMODE const s_aenmAddrMode[8] =
6631	{
6632	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
6633	};
6634	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
6635	if (enmAddrMode != (IEMMODE)-1)
6636	{
6637	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6638	if (cbInstr <= 15 && cbInstr >= 1)
6639	{
6640	Assert(cbInstr >= 1U + IoExitInfo.n.u1REP);
6641	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6642	{
6643	/* Don't know exactly how to detect whether u3SEG is valid, currently
6644	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
6645	2384 Opterons when only checking NRIP. */
6646	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6647	if ( fSupportsNextRipSave
6648	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
6649	{
6650	AssertMsg(IoExitInfo.n.u3SEG == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1REP,
6651	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3SEG, cbInstr, IoExitInfo.n.u1REP));
6652	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6653	IoExitInfo.n.u3SEG, true /fIoChecked/);
6654	}
6655	else if (cbInstr == 1U + IoExitInfo.n.u1REP)
6656	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6657	X86_SREG_DS, true /fIoChecked/);
6658	else
6659	rcStrict = IEMExecOne(pVCpu);
6660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6661	}
6662	else
6663	{
6664	AssertMsg(IoExitInfo.n.u3SEG == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3SEG));
6665	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6666	true /fIoChecked/);
6667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6668	}
6669	}
6670	else
6671	{
6672	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
6673	rcStrict = IEMExecOne(pVCpu);
6674	}
6675	}
6676	else
6677	{
6678	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
6679	rcStrict = IEMExecOne(pVCpu);
6680	}
6681	fUpdateRipAlready = true;
6682
6683	#else
6684	/* INS/OUTS - I/O String instruction. */
6685	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
6686
6687	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
6688	* in EXITINFO1? Investigate once this thing is up and running. */
6689
6690	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
6691	if (rcStrict == VINF_SUCCESS)
6692	{
6693	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6694	{
6695	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6696	(DISCPUMODE)pDis->uAddrMode, cbValue);
6697	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6698	}
6699	else
6700	{
6701	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6702	(DISCPUMODE)pDis->uAddrMode, cbValue);
6703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6704	}
6705	}
6706	else
6707	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
6708	#endif
6709	}
6710	else
6711	{
6712	/* IN/OUT - I/O instruction. */
6713	Assert(!IoExitInfo.n.u1REP);
6714
6715	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6716	{
6717	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
6718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
6719	}
6720	else
6721	{
6722	uint32_t u32Val = 0;
6723	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
6724	if (IOM_SUCCESS(rcStrict))
6725	{
6726	/* Save result of I/O IN instr. in AL/AX/EAX. */
6727	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
6728	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
6729	}
6730	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6731	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
6732
6733	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
6734	}
6735	}
6736
6737	if (IOM_SUCCESS(rcStrict))
6738	{
6739	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
6740	if (!fUpdateRipAlready)
6741	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
6742
6743	/*
6744	* If any I/O breakpoints are armed, we need to check if one triggered
6745	* and take appropriate action.
6746	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
6747	*/
6748	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
6749	* execution engines about whether hyper BPs and such are pending. */
6750	uint32_t const uDr7 = pCtx->dr[7];
6751	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
6752	&& X86_DR7_ANY_RW_IO(uDr7)
6753	&& (pCtx->cr4 & X86_CR4_DE))
6754	\|\| DBGFBpIsHwIoArmed(pVM)))
6755	{
6756	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6757	VMMRZCallRing3Disable(pVCpu);
6758	HM_DISABLE_PREEMPT();
6759
6760	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
6761	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
6762
6763	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
6764	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
6765	{
6766	/* Raise #DB. */
6767	pVmcb->guest.u64DR6 = pCtx->dr[6];
6768	pVmcb->guest.u64DR7 = pCtx->dr[7];
6769	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
6770	hmR0SvmSetPendingXcptDB(pVCpu);
6771	}
6772	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
6773	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
6774	else if ( rcStrict2 != VINF_SUCCESS
6775	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
6776	rcStrict = rcStrict2;
6777	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
6778
6779	HM_RESTORE_PREEMPT();
6780	VMMRZCallRing3Enable(pVCpu);
6781	}
6782
6783	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6784	}
6785
6786	#ifdef VBOX_STRICT
6787	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6788	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
6789	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
6790	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
6791	else
6792	{
6793	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
6794	* statuses, that the VMM device and some others may return. See
6795	* IOM_SUCCESS() for guidance. */
6796	AssertMsg( RT_FAILURE(rcStrict)
6797	\|\| rcStrict == VINF_SUCCESS
6798	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
6799	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
6800	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
6801	\|\| rcStrict == VINF_EM_RAW_TO_R3
6802	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED
6803	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6804	}
6805	#endif
6806	return VBOXSTRICTRC_TODO(rcStrict);
6807	}
6808
6809
6810	/**
6811	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
6812	*/
6813	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6814	{
6815	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6816	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
6817
6818	PVM pVM = pVCpu->CTX_SUFF(pVM);
6819	Assert(pVM->hm.s.fNestedPaging);
6820
6821	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
6822
6823	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
6824	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6825	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
6826	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
6827
6828	Log4(("#NPF at CS:RIP=%04x:%#RX64 faultaddr=%RGp errcode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr, u32ErrCode));
6829
6830	#ifdef VBOX_HM_WITH_GUEST_PATCHING
6831	/* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions. */
6832	if ( pVM->hm.s.fTprPatchingAllowed
6833	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
6834	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
6835	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
6836	&& !CPUMIsGuestInLongModeEx(pCtx)
6837	&& !CPUMGetGuestCPL(pVCpu)
6838	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
6839	{
6840	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
6841	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6842
6843	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
6844	{
6845	/* Only attempt to patch the instruction once. */
6846	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
6847	if (!pPatch)
6848	return VINF_EM_HM_PATCH_TPR_INSTR;
6849	}
6850	}
6851	#endif
6852
6853	/*
6854	* Determine the nested paging mode.
6855	*/
6856	PGMMODE enmNestedPagingMode;
6857	#if HC_ARCH_BITS == 32
6858	if (CPUMIsGuestInLongModeEx(pCtx))
6859	enmNestedPagingMode = PGMMODE_AMD64_NX;
6860	else
6861	#endif
6862	enmNestedPagingMode = PGMGetHostMode(pVM);
6863
6864	/*
6865	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
6866	*/
6867	int rc;
6868	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
6869	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
6870	{
6871	/* If event delivery causes an MMIO #NPF, go back to instruction emulation as
6872	otherwise injecting the original pending event would most likely cause the same MMIO #NPF. */
6873	if (pVCpu->hm.s.Event.fPending)
6874	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6875
6876	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
6877	u32ErrCode);
6878	rc = VBOXSTRICTRC_VAL(rc2);
6879
6880	/*
6881	* If we succeed, resume guest execution.
6882	* If we fail in interpreting the instruction because we couldn't get the guest physical address
6883	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
6884	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
6885	* weird case. See @bugref{6043}.
6886	*/
6887	if ( rc == VINF_SUCCESS
6888	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
6889	\|\| rc == VERR_PAGE_NOT_PRESENT)
6890	{
6891	/* Successfully handled MMIO operation. */
6892	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6893	rc = VINF_SUCCESS;
6894	}
6895	return rc;
6896	}
6897
6898	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
6899	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
6900	TRPMResetTrap(pVCpu);
6901
6902	Log4(("#NPF: PGMR0Trap0eHandlerNestedPaging returned %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
6903
6904	/*
6905	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
6906	*/
6907	if ( rc == VINF_SUCCESS
6908	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
6909	\|\| rc == VERR_PAGE_NOT_PRESENT)
6910	{
6911	/* We've successfully synced our shadow page tables. */
6912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
6913	rc = VINF_SUCCESS;
6914	}
6915
6916	return rc;
6917	}
6918
6919
6920	/**
6921	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
6922	* \#VMEXIT.
6923	*/
6924	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6925	{
6926	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6927	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
6928
6929	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6930	hmR0SvmClearVirtIntrIntercept(pVmcb);
6931
6932	/* Deliver the pending interrupt/NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
6933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6934	return VINF_SUCCESS;
6935	}
6936
6937
6938	/**
6939	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
6940	* \#VMEXIT.
6941	*/
6942	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6943	{
6944	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6945
6946	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
6947
6948	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
6949	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
6950	#endif
6951
6952	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
6953	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
6954	{
6955	/*
6956	* AMD-V provides us with the exception which caused the TS; we collect
6957	* the information in the call to hmR0SvmCheckExitDueToEventDelivery.
6958	*/
6959	Log4(("hmR0SvmExitTaskSwitch: TS occurred during event delivery.\n"));
6960	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
6961	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6962	}
6963
6964	/** @todo Emulate task switch someday, currently just going back to ring-3 for
6965	* emulation. */
6966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
6967	return VERR_EM_INTERPRETER;
6968	}
6969
6970
6971	/**
6972	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
6973	*/
6974	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6975	{
6976	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
6978
6979	bool fRipUpdated;
6980	VBOXSTRICTRC rcStrict = HMSvmVmmcall(pVCpu, pCtx, &fRipUpdated);
6981	if (RT_SUCCESS(rcStrict))
6982	{
6983	/* Only update the RIP if we're continuing guest execution and not
6984	in the case of say VINF_GIM_R3_HYPERCALL. */
6985	if ( rcStrict == VINF_SUCCESS
6986	&& !fRipUpdated)
6987	{
6988	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3 /* cbInstr */);
6989	}
6990
6991	/* If the hypercall or TPR patching changes anything other than guest's general-purpose registers,
6992	we would need to reload the guest changed bits here before VM-entry. */
6993	return VBOXSTRICTRC_VAL(rcStrict);
6994	}
6995
6996	hmR0SvmSetPendingXcptUD(pVCpu);
6997	return VINF_SUCCESS;
6998	}
6999
7000
7001	/**
7002	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7003	*/
7004	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7005	{
7006	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7008	return VINF_EM_RAW_INTERRUPT;
7009	}
7010
7011
7012	/**
7013	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7014	*/
7015	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7016	{
7017	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7018
7019	/* Clear NMI blocking. */
7020	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7021
7022	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7023	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7024	hmR0SvmClearIretIntercept(pVmcb);
7025
7026	/* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7027	return VINF_SUCCESS;
7028	}
7029
7030
7031	/**
7032	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_EXCEPTION_14).
7033	* Conditional \#VMEXIT.
7034	*/
7035	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7036	{
7037	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7038	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7039
7040	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7041
7042	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7043	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7044	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
7045	RTGCUINTPTR uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7046	PVM pVM = pVCpu->CTX_SUFF(pVM);
7047
7048	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
7049	if (pVM->hm.s.fNestedPaging)
7050	{
7051	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7052	if (!pSvmTransient->fVectoringDoublePF)
7053	{
7054	/* A genuine guest #PF, reflect it to the guest. */
7055	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7056	Log4(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RGv ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7057	uFaultAddress, u32ErrCode));
7058	}
7059	else
7060	{
7061	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7062	hmR0SvmSetPendingXcptDF(pVCpu);
7063	Log4(("Pending #DF due to vectoring #PF. NP\n"));
7064	}
7065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7066	return VINF_SUCCESS;
7067	}
7068	#endif
7069
7070	Assert(!pVM->hm.s.fNestedPaging);
7071
7072	#ifdef VBOX_HM_WITH_GUEST_PATCHING
7073	/* Shortcut for APIC TPR reads and writes; only applicable to 32-bit guests. */
7074	if ( pVM->hm.s.fTprPatchingAllowed
7075	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7076	&& !(u32ErrCode & X86_TRAP_PF_P) /* Not present. */
7077	&& !CPUMIsGuestInLongModeEx(pCtx)
7078	&& !CPUMGetGuestCPL(pVCpu)
7079	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7080	{
7081	RTGCPHYS GCPhysApicBase;
7082	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7083	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7084
7085	/* Check if the page at the fault-address is the APIC base. */
7086	RTGCPHYS GCPhysPage;
7087	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7088	if ( rc2 == VINF_SUCCESS
7089	&& GCPhysPage == GCPhysApicBase)
7090	{
7091	/* Only attempt to patch the instruction once. */
7092	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7093	if (!pPatch)
7094	return VINF_EM_HM_PATCH_TPR_INSTR;
7095	}
7096	}
7097	#endif
7098
7099	Log4(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 u32ErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7100	pCtx->rip, u32ErrCode, pCtx->cr3));
7101
7102	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7103	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7104	if (pSvmTransient->fVectoringPF)
7105	{
7106	Assert(pVCpu->hm.s.Event.fPending);
7107	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7108	}
7109
7110	TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
7111	int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7112
7113	Log4(("#PF rc=%Rrc\n", rc));
7114
7115	if (rc == VINF_SUCCESS)
7116	{
7117	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7118	TRPMResetTrap(pVCpu);
7119	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7120	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7121	return rc;
7122	}
7123	else if (rc == VINF_EM_RAW_GUEST_TRAP)
7124	{
7125	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7126
7127	if (!pSvmTransient->fVectoringDoublePF)
7128	{
7129	/* It's a guest page fault and needs to be reflected to the guest. */
7130	u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7131	TRPMResetTrap(pVCpu);
7132	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7133	}
7134	else
7135	{
7136	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7137	TRPMResetTrap(pVCpu);
7138	hmR0SvmSetPendingXcptDF(pVCpu);
7139	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7140	}
7141
7142	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7143	return VINF_SUCCESS;
7144	}
7145
7146	TRPMResetTrap(pVCpu);
7147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7148	return rc;
7149	}
7150
7151
7152	/**
7153	* \#VMEXIT handler for device-not-available exceptions (SVM_EXIT_EXCEPTION_7).
7154	* Conditional \#VMEXIT.
7155	*/
7156	HMSVM_EXIT_DECL hmR0SvmExitXcptNM(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7157	{
7158	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7159
7160	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7161	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7162	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7163
7164	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7165	VMMRZCallRing3Disable(pVCpu);
7166	HM_DISABLE_PREEMPT();
7167
7168	int rc;
7169	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
7170	if (pSvmTransient->fWasGuestFPUStateActive)
7171	{
7172	rc = VINF_EM_RAW_GUEST_TRAP;
7173	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
7174	}
7175	else
7176	{
7177	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
7178	Assert(!pSvmTransient->fWasGuestFPUStateActive);
7179	#endif
7180	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu); /* (No need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
7181	Assert( rc == VINF_EM_RAW_GUEST_TRAP
7182	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
7183	}
7184
7185	HM_RESTORE_PREEMPT();
7186	VMMRZCallRing3Enable(pVCpu);
7187
7188	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
7189	{
7190	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
7191	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
7193	pVCpu->hm.s.fPreloadGuestFpu = true;
7194	}
7195	else
7196	{
7197	/* Forward #NM to the guest. */
7198	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
7199	hmR0SvmSetPendingXcptNM(pVCpu);
7200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
7201	}
7202	return VINF_SUCCESS;
7203	}
7204
7205
7206	/**
7207	* \#VMEXIT handler for undefined opcode (SVM_EXIT_EXCEPTION_6).
7208	* Conditional \#VMEXIT.
7209	*/
7210	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7211	{
7212	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7213
7214	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7215	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7216	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7217
7218	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7219	if (pVCpu->hm.s.fGIMTrapXcptUD)
7220	{
7221	uint8_t cbInstr = 0;
7222	VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, pCtx, NULL /* pDis */, &cbInstr);
7223	if (rcStrict == VINF_SUCCESS)
7224	{
7225	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7226	hmR0SvmAdvanceRipDumb(pVCpu, pCtx, cbInstr);
7227	rc = VINF_SUCCESS;
7228	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7229	}
7230	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7231	rc = VINF_SUCCESS;
7232	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7233	rc = VINF_GIM_R3_HYPERCALL;
7234	else
7235	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7236	}
7237
7238	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7239	if (RT_FAILURE(rc))
7240	{
7241	hmR0SvmSetPendingXcptUD(pVCpu);
7242	rc = VINF_SUCCESS;
7243	}
7244
7245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7246	return rc;
7247	}
7248
7249
7250	/**
7251	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_EXCEPTION_16).
7252	* Conditional \#VMEXIT.
7253	*/
7254	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7255	{
7256	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7257
7258	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7259	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7260	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7261
7262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7263
7264	if (!(pCtx->cr0 & X86_CR0_NE))
7265	{
7266	PVM pVM = pVCpu->CTX_SUFF(pVM);
7267	PDISSTATE pDis = &pVCpu->hm.s.DisState;
7268	unsigned cbOp;
7269	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
7270	if (RT_SUCCESS(rc))
7271	{
7272	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7273	/** @todo FERR intercept when in nested-guest mode? */
7274	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
7275	if (RT_SUCCESS(rc))
7276	pCtx->rip += cbOp;
7277	}
7278	else
7279	Log4(("hmR0SvmExitXcptMF: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7280	return rc;
7281	}
7282
7283	hmR0SvmSetPendingXcptMF(pVCpu);
7284	return VINF_SUCCESS;
7285	}
7286
7287
7288	/**
7289	* \#VMEXIT handler for debug exceptions (SVM_EXIT_EXCEPTION_1). Conditional
7290	* \#VMEXIT.
7291	*/
7292	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7293	{
7294	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7295
7296	/* If this #DB is the result of delivering an event, go back to the interpreter. */
7297	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7298	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7299	{
7300	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7301	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7302	}
7303
7304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7305
7306	/* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data breakpoint). However, for both cases
7307	DR6 and DR7 are updated to what the exception handler expects. See AMD spec. 15.12.2 "#DB (Debug)". */
7308	PVM pVM = pVCpu->CTX_SUFF(pVM);
7309	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7310	int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7311	if (rc == VINF_EM_RAW_GUEST_TRAP)
7312	{
7313	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7314	if (CPUMIsHyperDebugStateActive(pVCpu))
7315	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
7316
7317	/* Reflect the exception back to the guest. */
7318	hmR0SvmSetPendingXcptDB(pVCpu);
7319	rc = VINF_SUCCESS;
7320	}
7321
7322	/*
7323	* Update DR6.
7324	*/
7325	if (CPUMIsHyperDebugStateActive(pVCpu))
7326	{
7327	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7328	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7329	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7330	}
7331	else
7332	{
7333	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7334	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7335	}
7336
7337	return rc;
7338	}
7339
7340
7341	/**
7342	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_EXCEPTION_17).
7343	* Conditional \#VMEXIT.
7344	*/
7345	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7346	{
7347	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7348
7349	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7350	* shutdown VMEXIT. */
7351	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7352
7353	SVMEVENT Event;
7354	Event.u = 0;
7355	Event.n.u1Valid = 1;
7356	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7357	Event.n.u8Vector = X86_XCPT_AC;
7358	Event.n.u1ErrorCodeValid = 1;
7359	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7360	return VINF_SUCCESS;
7361	}
7362
7363
7364	/**
7365	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_EXCEPTION_3).
7366	* Conditional \#VMEXIT.
7367	*/
7368	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7369	{
7370	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7371
7372	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7373
7374	int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7375	if (rc == VINF_EM_RAW_GUEST_TRAP)
7376	{
7377	SVMEVENT Event;
7378	Event.u = 0;
7379	Event.n.u1Valid = 1;
7380	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7381	Event.n.u8Vector = X86_XCPT_BP;
7382	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7383	}
7384
7385	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
7386	return rc;
7387	}
7388
7389
7390	#ifdef VBOX_WITH_NESTED_HWVIRT
7391	/**
7392	* \#VMEXIT handler for #PF occuring while in nested-guest execution
7393	* (SVM_EXIT_EXCEPTION_14). Conditional \#VMEXIT.
7394	*/
7395	HMSVM_EXIT_DECL hmR0SvmExitXcptPFNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7396	{
7397	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7398
7399	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7400
7401	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7402	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7403	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
7404	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7405
7406	Log4(("#PFNested: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 u32ErrCode=%#RX32 CR3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7407	pCtx->rip, u32ErrCode, pCtx->cr3));
7408
7409	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7410	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7411	if (pSvmTransient->fVectoringPF)
7412	{
7413	Assert(pVCpu->hm.s.Event.fPending);
7414	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7415	}
7416
7417	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
7418
7419	TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
7420	int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7421
7422	Log4(("#PFNested: rc=%Rrc\n", rc));
7423
7424	if (rc == VINF_SUCCESS)
7425	{
7426	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7427	TRPMResetTrap(pVCpu);
7428	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7429	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7430	return rc;
7431	}
7432
7433	if (rc == VINF_EM_RAW_GUEST_TRAP)
7434	{
7435	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7436
7437	if (!pSvmTransient->fVectoringDoublePF)
7438	{
7439	/* It's a nested-guest page fault and needs to be reflected to the nested-guest. */
7440	u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7441	TRPMResetTrap(pVCpu);
7442	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7443	}
7444	else
7445	{
7446	/* A nested-guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7447	TRPMResetTrap(pVCpu);
7448	hmR0SvmSetPendingXcptDF(pVCpu);
7449	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7450	}
7451
7452	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7453	return VINF_SUCCESS;
7454	}
7455
7456	TRPMResetTrap(pVCpu);
7457	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7458	return rc;
7459	}
7460
7461
7462	/**
7463	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7464	*/
7465	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7466	{
7467	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7468
7469	/** @todo Stat. */
7470	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClgi); */
7471	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7472	VBOXSTRICTRC rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7473
7474	/*
7475	* The guest should no longer receive interrupts. Until VGIF is supported,
7476	* clear virtual interrupt intercepts here.
7477	*/
7478	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7479	hmR0SvmClearVirtIntrIntercept(pVmcb);
7480
7481	return VBOXSTRICTRC_VAL(rcStrict);
7482	}
7483
7484
7485	/**
7486	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7487	*/
7488	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7489	{
7490	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7491
7492	/** @todo Stat. */
7493	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitStgi); */
7494	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7495	VBOXSTRICTRC rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7496	return VBOXSTRICTRC_VAL(rcStrict);
7497	}
7498
7499
7500	/**
7501	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7502	*/
7503	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7504	{
7505	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7506
7507	/** @todo Stat. */
7508	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmload); */
7509	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7510	VBOXSTRICTRC rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7511	if (rcStrict == VINF_SUCCESS)
7512	{
7513	/* We skip flagging changes made to LSTAR, STAR, SFMASK and other MSRs as they are always re-loaded. */
7514	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7515	\| HM_CHANGED_GUEST_TR
7516	\| HM_CHANGED_GUEST_LDTR);
7517	}
7518	return VBOXSTRICTRC_VAL(rcStrict);
7519	}
7520
7521
7522	/**
7523	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7524	*/
7525	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7526	{
7527	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7528
7529	/** @todo Stat. */
7530	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmsave); */
7531	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7532	VBOXSTRICTRC rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7533	return VBOXSTRICTRC_VAL(rcStrict);
7534	}
7535
7536
7537	/**
7538	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
7539	*/
7540	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7541	{
7542	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7543	/** @todo Stat. */
7544	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpga); */
7545	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7546	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
7547	return VBOXSTRICTRC_VAL(rcStrict);
7548	}
7549
7550
7551	/**
7552	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
7553	*/
7554	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7555	{
7556	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7557	/** @todo Stat. */
7558	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmrun); */
7559	#if 0
7560	VBOXSTRICTRC rcStrict;
7561	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7562	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
7563	Log4(("IEMExecDecodedVmrun: returned %d\n", VBOXSTRICTRC_VAL(rcStrict)));
7564	if (rcStrict == VINF_SUCCESS)
7565	{
7566	rcStrict = VINF_SVM_VMRUN;
7567	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7568	}
7569	return VBOXSTRICTRC_VAL(rcStrict);
7570	#endif
7571	return VERR_EM_INTERPRETER;
7572	}
7573
7574
7575	/**
7576	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7577	*/
7578	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7579	{
7580	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7581
7582	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7583	* shutdown VMEXIT. */
7584	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7585
7586	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7587	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_EXCEPTION_0;
7588	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7589	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7590	Assert(uVector <= X86_XCPT_LAST);
7591
7592	SVMEVENT Event;
7593	Event.u = 0;
7594	Event.n.u1Valid = 1;
7595	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7596	Event.n.u8Vector = uVector;
7597	switch (uVector)
7598	{
7599	case X86_XCPT_PF:
7600	case X86_XCPT_DF:
7601	case X86_XCPT_TS:
7602	case X86_XCPT_NP:
7603	case X86_XCPT_SS:
7604	case X86_XCPT_GP:
7605	case X86_XCPT_AC:
7606	{
7607	Event.n.u1ErrorCodeValid = 1;
7608	Event.n.u32ErrorCode = uErrCode;
7609	break;
7610	}
7611	}
7612
7613	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7614	return VINF_SUCCESS;
7615	}
7616
7617
7618	/**
7619	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_EXCEPTION_1).
7620	* Unconditional \#VMEXIT.
7621	*/
7622	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7623	{
7624	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7625
7626	/* If this #DB is the result of delivering an event, go back to the interpreter. */
7627	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7628	* shutdown VMEXIT. */
7629	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7630	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7631	{
7632	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7633	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7634	}
7635
7636	hmR0SvmSetPendingXcptDB(pVCpu);
7637	return VINF_SUCCESS;
7638	}
7639
7640
7641	/**
7642	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_EXCEPTION_3).
7643	* Conditional \#VMEXIT.
7644	*/
7645	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7646	{
7647	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7648
7649	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7650	* shutdown VMEXIT. */
7651	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7652
7653	SVMEVENT Event;
7654	Event.u = 0;
7655	Event.n.u1Valid = 1;
7656	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7657	Event.n.u8Vector = X86_XCPT_BP;
7658	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7659	return VINF_SUCCESS;
7660	}
7661
7662	#endif /* VBOX_WITH_NESTED_HWVIRT */
7663
7664
7665	/** @} */
7666

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 70303

Download in other formats: