HMSVMR0.cpp@ 70262

Last change on this file since 70262 was 70262, checked in by vboxsync, 7 years ago
VMM/HM: SVM VMCB struct logical adjustments for new SVM features (Virtualized VMSAVE/VMLOAD).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 297.8 KB

Line
1	/* $Id: HMSVMR0.cpp 70262 2017-12-21 08:32:29Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/gim.h>
33	#include <VBox/vmm/apic.h>
34	#include "HMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include "HMSVMR0.h"
37	#include "dtrace/VBoxVMM.h"
38
39	#define HMSVM_USE_IEM_EVENT_REFLECTION
40	#ifdef DEBUG_ramshankar
41	# define HMSVM_SYNC_FULL_GUEST_STATE
42	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
43	# define HMSVM_ALWAYS_TRAP_PF
44	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
45	#endif
46
47
48	/*********************************************************************************************************************************
49	* Defined Constants And Macros *
50	*********************************************************************************************************************************/
51	#ifdef VBOX_WITH_STATISTICS
52	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
53	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
54	if ((u64ExitCode) == SVM_EXIT_NPF) \
55	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
56	else \
57	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
58	} while (0)
59	#else
60	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
61	#endif
62
63	/** If we decide to use a function table approach this can be useful to
64	* switch to a "static DECLCALLBACK(int)". */
65	#define HMSVM_EXIT_DECL static int
66
67	/** Macro for checking and returning from the using function for
68	* \#VMEXIT intercepts that maybe caused during delivering of another
69	* event in the guest. */
70	#define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
71	do \
72	{ \
73	int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
74	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* likely */ } \
75	else if (rc == VINF_HM_DOUBLE_FAULT) \
76	return VINF_SUCCESS; \
77	else \
78	return rc; \
79	} while (0)
80
81	/**
82	* Updates interrupt shadow for the current RIP.
83	*/
84	#define HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx) \
85	do { \
86	/* Update interrupt shadow. */ \
87	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS) \
88	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu)) \
89	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); \
90	} while (0)
91
92	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
93	* instruction that exited. */
94	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
95	do { \
96	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
97	(a_rc) = VINF_EM_DBG_STEPPED; \
98	} while (0)
99
100	/** Assert that preemption is disabled or covered by thread-context hooks. */
101	#define HMSVM_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
102	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
103
104	/** Assert that we haven't migrated CPUs when thread-context hooks are not
105	* used. */
106	#define HMSVM_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
107	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
108	("Illegal migration! Entered on CPU %u Current %u\n", \
109	pVCpu->hm.s.idEnteredCpu, RTMpCpuId()));
110
111	/** Assert that we're not executing a nested-guest. */
112	#ifdef VBOX_WITH_NESTED_HWVIRT
113	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
114	#else
115	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
116	#endif
117
118	/** Assert that we're executing a nested-guest. */
119	#ifdef VBOX_WITH_NESTED_HWVIRT
120	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
121	#else
122	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
123	#endif
124
125	/**
126	* Exception bitmap mask for all contributory exceptions.
127	*
128	* Page fault is deliberately excluded here as it's conditional as to whether
129	* it's contributory or benign. Page faults are handled separately.
130	*/
131	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
132	\| RT_BIT(X86_XCPT_DE))
133
134	/**
135	* Mandatory/unconditional guest control intercepts.
136	*
137	* SMIs can and do happen in normal operation. We need not intercept them
138	* while executing the guest or nested-guest.
139	*/
140	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
141	\| SVM_CTRL_INTERCEPT_NMI \
142	\| SVM_CTRL_INTERCEPT_INIT \
143	\| SVM_CTRL_INTERCEPT_RDPMC \
144	\| SVM_CTRL_INTERCEPT_CPUID \
145	\| SVM_CTRL_INTERCEPT_RSM \
146	\| SVM_CTRL_INTERCEPT_HLT \
147	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
148	\| SVM_CTRL_INTERCEPT_MSR_PROT \
149	\| SVM_CTRL_INTERCEPT_INVLPGA \
150	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
151	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
152	\| SVM_CTRL_INTERCEPT_VMRUN \
153	\| SVM_CTRL_INTERCEPT_VMMCALL \
154	\| SVM_CTRL_INTERCEPT_VMLOAD \
155	\| SVM_CTRL_INTERCEPT_VMSAVE \
156	\| SVM_CTRL_INTERCEPT_STGI \
157	\| SVM_CTRL_INTERCEPT_CLGI \
158	\| SVM_CTRL_INTERCEPT_SKINIT \
159	\| SVM_CTRL_INTERCEPT_WBINVD \
160	\| SVM_CTRL_INTERCEPT_MONITOR \
161	\| SVM_CTRL_INTERCEPT_MWAIT \
162	\| SVM_CTRL_INTERCEPT_XSETBV)
163
164	/** @name VMCB Clean Bits.
165	*
166	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
167	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
168	* memory.
169	*
170	* @{ */
171	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
172	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
173	/** I/O permission bitmap, MSR permission bitmap. */
174	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
175	/** ASID. */
176	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
177	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
178	V_INTR_VECTOR. */
179	#define HMSVM_VMCB_CLEAN_TPR RT_BIT(3)
180	/** Nested Paging: Nested CR3 (nCR3), PAT. */
181	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
182	/** Control registers (CR0, CR3, CR4, EFER). */
183	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
184	/** Debug registers (DR6, DR7). */
185	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
186	/** GDT, IDT limit and base. */
187	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
188	/** Segment register: CS, SS, DS, ES limit and base. */
189	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
190	/** CR2.*/
191	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
192	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
193	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
194	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
195	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
196	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
197	/** Mask of all valid VMCB Clean bits. */
198	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
199	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
200	\| HMSVM_VMCB_CLEAN_ASID \
201	\| HMSVM_VMCB_CLEAN_TPR \
202	\| HMSVM_VMCB_CLEAN_NP \
203	\| HMSVM_VMCB_CLEAN_CRX_EFER \
204	\| HMSVM_VMCB_CLEAN_DRX \
205	\| HMSVM_VMCB_CLEAN_DT \
206	\| HMSVM_VMCB_CLEAN_SEG \
207	\| HMSVM_VMCB_CLEAN_CR2 \
208	\| HMSVM_VMCB_CLEAN_LBR \
209	\| HMSVM_VMCB_CLEAN_AVIC)
210	/** @} */
211
212	/** @name SVM transient.
213	*
214	* A state structure for holding miscellaneous information across AMD-V
215	* VMRUN/\#VMEXIT operation, restored after the transition.
216	*
217	* @{ */
218	typedef struct SVMTRANSIENT
219	{
220	/** The host's rflags/eflags. */
221	RTCCUINTREG fEFlags;
222	#if HC_ARCH_BITS == 32
223	uint32_t u32Alignment0;
224	#endif
225
226	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
227	uint64_t u64ExitCode;
228	/** The guest's TPR value used for TPR shadowing. */
229	uint8_t u8GuestTpr;
230	/** Alignment. */
231	uint8_t abAlignment0[7];
232
233	/** Whether the guest FPU state was active at the time of \#VMEXIT. */
234	bool fWasGuestFPUStateActive;
235	/** Whether the guest debug state was active at the time of \#VMEXIT. */
236	bool fWasGuestDebugStateActive;
237	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
238	bool fWasHyperDebugStateActive;
239	/** Whether the TSC offset mode needs to be updated. */
240	bool fUpdateTscOffsetting;
241	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
242	bool fRestoreTscAuxMsr;
243	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
244	* contributary exception or a page-fault. */
245	bool fVectoringDoublePF;
246	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
247	* external interrupt or NMI. */
248	bool fVectoringPF;
249	} SVMTRANSIENT, *PSVMTRANSIENT;
250	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
251	AssertCompileMemberAlignment(SVMTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
252	/** @} */
253
254	/**
255	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
256	*/
257	typedef enum SVMMSREXITREAD
258	{
259	/** Reading this MSR causes a \#VMEXIT. */
260	SVMMSREXIT_INTERCEPT_READ = 0xb,
261	/** Reading this MSR does not cause a \#VMEXIT. */
262	SVMMSREXIT_PASSTHRU_READ
263	} SVMMSREXITREAD;
264
265	/**
266	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
267	*/
268	typedef enum SVMMSREXITWRITE
269	{
270	/** Writing to this MSR causes a \#VMEXIT. */
271	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
272	/** Writing to this MSR does not cause a \#VMEXIT. */
273	SVMMSREXIT_PASSTHRU_WRITE
274	} SVMMSREXITWRITE;
275
276	/**
277	* SVM \#VMEXIT handler.
278	*
279	* @returns VBox status code.
280	* @param pVCpu The cross context virtual CPU structure.
281	* @param pMixedCtx Pointer to the guest-CPU context.
282	* @param pSvmTransient Pointer to the SVM-transient structure.
283	*/
284	typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
285
286
287	/*********************************************************************************************************************************
288	* Internal Functions *
289	*********************************************************************************************************************************/
290	static void hmR0SvmSetMsrPermission(PSVMVMCB pVmcb, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
291	SVMMSREXITWRITE enmWrite);
292	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
293	static void hmR0SvmLeave(PVMCPU pVCpu);
294
295	/** @name \#VMEXIT handlers.
296	* @{
297	*/
298	static FNSVMEXITHANDLER hmR0SvmExitIntr;
299	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
300	static FNSVMEXITHANDLER hmR0SvmExitInvd;
301	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
302	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
303	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
304	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
305	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
306	static FNSVMEXITHANDLER hmR0SvmExitHlt;
307	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
308	static FNSVMEXITHANDLER hmR0SvmExitMwait;
309	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
310	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
311	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
312	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
313	static FNSVMEXITHANDLER hmR0SvmExitMsr;
314	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
315	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
316	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
317	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
318	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
319	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
320	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
321	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
322	static FNSVMEXITHANDLER hmR0SvmExitPause;
323	static FNSVMEXITHANDLER hmR0SvmExitIret;
324	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
325	static FNSVMEXITHANDLER hmR0SvmExitXcptNM;
326	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
327	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
328	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
329	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
330	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
331	#ifdef VBOX_WITH_NESTED_HWVIRT
332	static FNSVMEXITHANDLER hmR0SvmExitXcptPFNested;
333	static FNSVMEXITHANDLER hmR0SvmExitClgi;
334	static FNSVMEXITHANDLER hmR0SvmExitStgi;
335	static FNSVMEXITHANDLER hmR0SvmExitVmload;
336	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
337	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
338	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
339	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
340	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
341	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
342	#endif
343	/** @} */
344
345	static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient);
346	#ifdef VBOX_WITH_NESTED_HWVIRT
347	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
348	#endif
349
350
351	/*********************************************************************************************************************************
352	* Global Variables *
353	*********************************************************************************************************************************/
354	/** Ring-0 memory object for the IO bitmap. */
355	RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
356	/** Physical address of the IO bitmap. */
357	RTHCPHYS g_HCPhysIOBitmap = 0;
358	/** Pointer to the IO bitmap. */
359	R0PTRTYPE(void *) g_pvIOBitmap = NULL;
360
361	#ifdef VBOX_WITH_NESTED_HWVIRT
362	/** Ring-0 memory object for the nested-guest MSRPM bitmap. */
363	RTR0MEMOBJ g_hMemObjNstGstMsrBitmap = NIL_RTR0MEMOBJ;
364	/** Physical address of the nested-guest MSRPM bitmap. */
365	RTHCPHYS g_HCPhysNstGstMsrBitmap = 0;
366	/** Pointer to the nested-guest MSRPM bitmap. */
367	R0PTRTYPE(void *) g_pvNstGstMsrBitmap = NULL;
368	#endif
369
370	/**
371	* Sets up and activates AMD-V on the current CPU.
372	*
373	* @returns VBox status code.
374	* @param pCpu Pointer to the CPU info struct.
375	* @param pVM The cross context VM structure. Can be
376	* NULL after a resume!
377	* @param pvCpuPage Pointer to the global CPU page.
378	* @param HCPhysCpuPage Physical address of the global CPU page.
379	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
380	* @param pvArg Unused on AMD-V.
381	*/
382	VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
383	void *pvArg)
384	{
385	Assert(!fEnabledByHost);
386	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
387	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
388	Assert(pvCpuPage); NOREF(pvCpuPage);
389	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
390
391	NOREF(pvArg);
392	NOREF(fEnabledByHost);
393
394	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
395	RTCCUINTREG fEFlags = ASMIntDisableFlags();
396
397	/*
398	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
399	*/
400	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
401	if (u64HostEfer & MSR_K6_EFER_SVME)
402	{
403	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
404	if ( pVM
405	&& pVM->hm.s.svm.fIgnoreInUseError)
406	{
407	pCpu->fIgnoreAMDVInUseError = true;
408	}
409
410	if (!pCpu->fIgnoreAMDVInUseError)
411	{
412	ASMSetFlags(fEFlags);
413	return VERR_SVM_IN_USE;
414	}
415	}
416
417	/* Turn on AMD-V in the EFER MSR. */
418	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
419
420	/* Write the physical page address where the CPU will store the host state while executing the VM. */
421	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
422
423	/* Restore interrupts. */
424	ASMSetFlags(fEFlags);
425
426	/*
427	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all non-zero ASIDs
428	* when enabling SVM. AMD doesn't have an SVM instruction to flush all ASIDs (flushing is done
429	* upon VMRUN). Therefore, flag that we need to flush the TLB entirely with before executing any
430	* guest code.
431	*/
432	pCpu->fFlushAsidBeforeUse = true;
433
434	/*
435	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
436	*/
437	++pCpu->cTlbFlushes;
438
439	return VINF_SUCCESS;
440	}
441
442
443	/**
444	* Deactivates AMD-V on the current CPU.
445	*
446	* @returns VBox status code.
447	* @param pCpu Pointer to the CPU info struct.
448	* @param pvCpuPage Pointer to the global CPU page.
449	* @param HCPhysCpuPage Physical address of the global CPU page.
450	*/
451	VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
452	{
453	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
454	AssertReturn( HCPhysCpuPage
455	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
456	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
457	NOREF(pCpu);
458
459	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
460	RTCCUINTREG fEFlags = ASMIntDisableFlags();
461
462	/* Turn off AMD-V in the EFER MSR. */
463	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
464	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
465
466	/* Invalidate host state physical address. */
467	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
468
469	/* Restore interrupts. */
470	ASMSetFlags(fEFlags);
471
472	return VINF_SUCCESS;
473	}
474
475
476	/**
477	* Does global AMD-V initialization (called during module initialization).
478	*
479	* @returns VBox status code.
480	*/
481	VMMR0DECL(int) SVMR0GlobalInit(void)
482	{
483	/*
484	* Allocate 12 KB for the IO bitmap. Since this is non-optional and we always intercept all IO accesses, it's done
485	* once globally here instead of per-VM.
486	*/
487	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
488	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
489	if (RT_FAILURE(rc))
490	return rc;
491
492	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
493	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
494
495	/* Set all bits to intercept all IO accesses. */
496	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
497
498	#ifdef VBOX_WITH_NESTED_HWVIRT
499	/*
500	* Allocate 8 KB for the MSR permission bitmap for the nested-guest.
501	*/
502	Assert(g_hMemObjNstGstMsrBitmap == NIL_RTR0MEMOBJ);
503	rc = RTR0MemObjAllocCont(&g_hMemObjNstGstMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
504	if (RT_FAILURE(rc))
505	return rc;
506
507	g_pvNstGstMsrBitmap = RTR0MemObjAddress(g_hMemObjNstGstMsrBitmap);
508	g_HCPhysNstGstMsrBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjNstGstMsrBitmap, 0 /* iPage */);
509
510	/* Set all bits to intercept all MSR accesses. */
511	ASMMemFill32(g_pvNstGstMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
512	#endif
513
514	return VINF_SUCCESS;
515	}
516
517
518	/**
519	* Does global AMD-V termination (called during module termination).
520	*/
521	VMMR0DECL(void) SVMR0GlobalTerm(void)
522	{
523	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
524	{
525	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
526	g_pvIOBitmap = NULL;
527	g_HCPhysIOBitmap = 0;
528	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
529	}
530
531	#ifdef VBOX_WITH_NESTED_HWVIRT
532	if (g_hMemObjNstGstMsrBitmap != NIL_RTR0MEMOBJ)
533	{
534	RTR0MemObjFree(g_hMemObjNstGstMsrBitmap, true /* fFreeMappings */);
535	g_pvNstGstMsrBitmap = NULL;
536	g_HCPhysNstGstMsrBitmap = 0;
537	g_hMemObjNstGstMsrBitmap = NIL_RTR0MEMOBJ;
538	}
539	#endif
540	}
541
542
543	/**
544	* Frees any allocated per-VCPU structures for a VM.
545	*
546	* @param pVM The cross context VM structure.
547	*/
548	DECLINLINE(void) hmR0SvmFreeStructs(PVM pVM)
549	{
550	for (uint32_t i = 0; i < pVM->cCpus; i++)
551	{
552	PVMCPU pVCpu = &pVM->aCpus[i];
553	AssertPtr(pVCpu);
554
555	if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
556	{
557	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
558	pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
559	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
560	}
561
562	if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
563	{
564	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
565	pVCpu->hm.s.svm.pVmcb = NULL;
566	pVCpu->hm.s.svm.HCPhysVmcb = 0;
567	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
568	}
569
570	if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
571	{
572	RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
573	pVCpu->hm.s.svm.pvMsrBitmap = NULL;
574	pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
575	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
576	}
577	}
578	}
579
580
581	/**
582	* Does per-VM AMD-V initialization.
583	*
584	* @returns VBox status code.
585	* @param pVM The cross context VM structure.
586	*/
587	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
588	{
589	int rc = VERR_INTERNAL_ERROR_5;
590
591	/*
592	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
593	*/
594	uint32_t u32Family;
595	uint32_t u32Model;
596	uint32_t u32Stepping;
597	if (HMAmdIsSubjectToErratum170(&u32Family, &u32Model, &u32Stepping))
598	{
599	Log4(("SVMR0InitVM: AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
600	pVM->hm.s.svm.fAlwaysFlushTLB = true;
601	}
602
603	/*
604	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
605	*/
606	for (VMCPUID i = 0; i < pVM->cCpus; i++)
607	{
608	PVMCPU pVCpu = &pVM->aCpus[i];
609	pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
610	pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
611	pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
612	}
613
614	for (VMCPUID i = 0; i < pVM->cCpus; i++)
615	{
616	PVMCPU pVCpu = &pVM->aCpus[i];
617
618	/*
619	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
620	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
621	*/
622	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
623	if (RT_FAILURE(rc))
624	goto failure_cleanup;
625
626	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
627	pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
628	Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
629	ASMMemZeroPage(pvVmcbHost);
630
631	/*
632	* Allocate one page for the guest-state VMCB.
633	*/
634	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
635	if (RT_FAILURE(rc))
636	goto failure_cleanup;
637
638	pVCpu->hm.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
639	pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
640	Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
641	ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
642
643	/*
644	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
645	* SVM to not require one.
646	*/
647	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
648	false /* fExecutable */);
649	if (RT_FAILURE(rc))
650	goto failure_cleanup;
651
652	pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
653	pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
654	/* Set all bits to intercept all MSR accesses (changed later on). */
655	ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
656	}
657
658	return VINF_SUCCESS;
659
660	failure_cleanup:
661	hmR0SvmFreeStructs(pVM);
662	return rc;
663	}
664
665
666	/**
667	* Does per-VM AMD-V termination.
668	*
669	* @returns VBox status code.
670	* @param pVM The cross context VM structure.
671	*/
672	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
673	{
674	hmR0SvmFreeStructs(pVM);
675	return VINF_SUCCESS;
676	}
677
678
679	/**
680	* Returns whether the VMCB Clean Bits feature is supported.
681	*
682	* @return @c true if supported, @c false otherwise.
683	* @param pVCpu The cross context virtual CPU structure.
684	* @param pCtx Pointer to the guest-CPU context.
685	*/
686	DECLINLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPU pVCpu, PCPUMCTX pCtx)
687	{
688	PVM pVM = pVCpu->CTX_SUFF(pVM);
689	#ifdef VBOX_WITH_NESTED_HWVIRT
690	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
691	{
692	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN)
693	&& pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
694	}
695	#else
696	RT_NOREF(pCtx);
697	#endif
698	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
699	}
700
701
702	/**
703	* Returns whether the decode assists feature is supported.
704	*
705	* @return @c true if supported, @c false otherwise.
706	* @param pVCpu The cross context virtual CPU structure.
707	* @param pCtx Pointer to the guest-CPU context.
708	*/
709	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPU pVCpu, PCPUMCTX pCtx)
710	{
711	PVM pVM = pVCpu->CTX_SUFF(pVM);
712	#ifdef VBOX_WITH_NESTED_HWVIRT
713	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
714	{
715	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
716	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
717	}
718	#else
719	RT_NOREF(pCtx);
720	#endif
721	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
722	}
723
724
725	/**
726	* Returns whether the NRIP_SAVE feature is supported.
727	*
728	* @return @c true if supported, @c false otherwise.
729	* @param pVCpu The cross context virtual CPU structure.
730	* @param pCtx Pointer to the guest-CPU context.
731	*/
732	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPU pVCpu, PCPUMCTX pCtx)
733	{
734	PVM pVM = pVCpu->CTX_SUFF(pVM);
735	#ifdef VBOX_WITH_NESTED_HWVIRT
736	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
737	{
738	return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
739	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
740	}
741	#else
742	RT_NOREF(pCtx);
743	#endif
744	return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
745	}
746
747
748	/**
749	* Sets the permission bits for the specified MSR in the MSRPM.
750	*
751	* @param pVmcb Pointer to the VM control block.
752	* @param pbMsrBitmap Pointer to the MSR bitmap.
753	* @param uMsr The MSR for which the access permissions are being set.
754	* @param enmRead MSR read permissions.
755	* @param enmWrite MSR write permissions.
756	*/
757	static void hmR0SvmSetMsrPermission(PSVMVMCB pVmcb, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
758	SVMMSREXITWRITE enmWrite)
759	{
760	uint16_t offMsrpm;
761	uint32_t uMsrpmBit;
762	int rc = HMSvmGetMsrpmOffsetAndBit(uMsr, &offMsrpm, &uMsrpmBit);
763	AssertRC(rc);
764
765	Assert(uMsrpmBit < 0x3fff);
766	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
767
768	pbMsrBitmap += offMsrpm;
769	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
770	ASMBitSet(pbMsrBitmap, uMsrpmBit);
771	else
772	ASMBitClear(pbMsrBitmap, uMsrpmBit);
773
774	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
775	ASMBitSet(pbMsrBitmap, uMsrpmBit + 1);
776	else
777	ASMBitClear(pbMsrBitmap, uMsrpmBit + 1);
778
779	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
780	}
781
782
783	/**
784	* Sets up AMD-V for the specified VM.
785	* This function is only called once per-VM during initalization.
786	*
787	* @returns VBox status code.
788	* @param pVM The cross context VM structure.
789	*/
790	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
791	{
792	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
793	AssertReturn(pVM, VERR_INVALID_PARAMETER);
794	Assert(pVM->hm.s.svm.fSupported);
795
796	bool const fPauseFilter = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
797	bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
798	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter && pVM->hm.s.svm.cPauseFilterThresholdTicks;
799
800	for (VMCPUID i = 0; i < pVM->cCpus; i++)
801	{
802	PVMCPU pVCpu = &pVM->aCpus[i];
803	PSVMVMCB pVmcb = pVM->aCpus[i].hm.s.svm.pVmcb;
804
805	AssertMsgReturn(pVmcb, ("Invalid pVmcb for vcpu[%u]\n", i), VERR_SVM_INVALID_PVMCB);
806
807	/* Initialize the #VMEXIT history array with end-of-array markers (UINT16_MAX). */
808	Assert(!pVCpu->hm.s.idxExitHistoryFree);
809	HMCPU_EXIT_HISTORY_RESET(pVCpu);
810
811	/* Always trap #AC for reasons of security. */
812	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
813
814	/* Always trap #DB for reasons of security. */
815	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
816
817	/* Trap exceptions unconditionally (debug purposes). */
818	#ifdef HMSVM_ALWAYS_TRAP_PF
819	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
820	#endif
821	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
822	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
823	pVmcb->ctrl.u32InterceptXcpt \|= 0
824	\| RT_BIT(X86_XCPT_BP)
825	\| RT_BIT(X86_XCPT_DE)
826	\| RT_BIT(X86_XCPT_NM)
827	\| RT_BIT(X86_XCPT_UD)
828	\| RT_BIT(X86_XCPT_NP)
829	\| RT_BIT(X86_XCPT_SS)
830	\| RT_BIT(X86_XCPT_GP)
831	\| RT_BIT(X86_XCPT_PF)
832	\| RT_BIT(X86_XCPT_MF)
833	;
834	#endif
835
836	/* Set up unconditional intercepts and conditions. */
837	pVmcb->ctrl.u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
838
839	/* CR0, CR4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
840	pVmcb->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
841
842	/* CR0, CR4 writes must be intercepted for the same reasons as above. */
843	pVmcb->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4);
844
845	/* Intercept all DRx reads and writes by default. Changed later on. */
846	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
847	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
848
849	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
850	pVmcb->ctrl.IntCtrl.n.u1VIntrMasking = 1;
851
852	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
853	and we currently deliver both PIC and APIC interrupts alike. See hmR0SvmInjectPendingEvent() */
854	pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
855
856	/* Set IO and MSR bitmap permission bitmap physical addresses. */
857	pVmcb->ctrl.u64IOPMPhysAddr = g_HCPhysIOBitmap;
858	pVmcb->ctrl.u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
859
860	/* No LBR virtualization. */
861	Assert(pVmcb->ctrl.u1LbrVirt == 0);
862
863	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
864	Assert(pVmcb->ctrl.u32VmcbCleanBits == 0);
865
866	/* The host ASID MBZ, for the guest start with 1. */
867	pVmcb->ctrl.TLBCtrl.n.u32ASID = 1;
868
869	/*
870	* Setup the PAT MSR (applicable for Nested Paging only).
871	* The default value should be 0x0007040600070406ULL, but we want to treat all guest memory as WB,
872	* so choose type 6 for all PAT slots.
873	*/
874	pVmcb->guest.u64GPAT = UINT64_C(0x0006060606060606);
875
876	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
877	pVmcb->ctrl.u1NestedPaging = pVM->hm.s.fNestedPaging;
878
879	/* Without Nested Paging, we need additionally intercepts. */
880	if (!pVM->hm.s.fNestedPaging)
881	{
882	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
883	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(3);
884	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(3);
885
886	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
887	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
888	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
889
890	/* Page faults must be intercepted to implement shadow paging. */
891	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
892	}
893
894	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
895	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
896	#endif
897
898	/* Apply the exceptions intercepts needed by the GIM provider. */
899	if (pVCpu->hm.s.fGIMTrapXcptUD)
900	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
901
902	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
903	if (fUsePauseFilter)
904	{
905	pVmcb->ctrl.u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
906	if (fPauseFilterThreshold)
907	pVmcb->ctrl.u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
908	}
909
910	/*
911	* The following MSRs are saved/restored automatically during the world-switch.
912	* Don't intercept guest read/write accesses to these MSRs.
913	*/
914	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
915	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
916	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
917	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
918	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
919	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
920	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
921	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
922	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
923	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
924	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
925	}
926
927	return VINF_SUCCESS;
928	}
929
930
931	/**
932	* Gets a pointer to the currently active guest or nested-guest VMCB.
933	*
934	* @returns Pointer to the current context VMCB.
935	* @param pVCpu The cross context virtual CPU structure.
936	* @param pCtx Pointer to the guest-CPU context.
937	*/
938	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
939	{
940	#ifdef VBOX_WITH_NESTED_HWVIRT
941	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
942	return pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
943	#else
944	RT_NOREF(pCtx);
945	#endif
946	return pVCpu->hm.s.svm.pVmcb;
947	}
948
949
950	/**
951	* Invalidates a guest page by guest virtual address.
952	*
953	* @returns VBox status code.
954	* @param pVM The cross context VM structure.
955	* @param pVCpu The cross context virtual CPU structure.
956	* @param GCVirt Guest virtual address of the page to invalidate.
957	*/
958	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
959	{
960	AssertReturn(pVM, VERR_INVALID_PARAMETER);
961	Assert(pVM->hm.s.svm.fSupported);
962
963	bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB \|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
964
965	/* Skip it if a TLB flush is already pending. */
966	if (!fFlushPending)
967	{
968	Log4(("SVMR0InvalidatePage %RGv\n", GCVirt));
969
970	PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
971	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
972	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
973
974	#if HC_ARCH_BITS == 32
975	/* If we get a flush in 64-bit guest mode, then force a full TLB flush. INVLPGA takes only 32-bit addresses. */
976	if (CPUMIsGuestInLongMode(pVCpu))
977	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
978	else
979	#endif
980	{
981	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
982	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
983	}
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Flushes the appropriate tagged-TLB entries.
991	*
992	* @param pVCpu The cross context virtual CPU structure.
993	* @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
994	* @param pVmcb Pointer to the VM control block.
995	*/
996	static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
997	{
998	#ifndef VBOX_WITH_NESTED_HWVIRT
999	RT_NOREF(pCtx);
1000	#endif
1001
1002	PVM pVM = pVCpu->CTX_SUFF(pVM);
1003	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
1004
1005	/*
1006	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1007	* This can happen both for start & resume due to long jumps back to ring-3.
1008	*
1009	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no correlation
1010	* between it and the physical CPU.
1011	*
1012	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
1013	* so we cannot reuse the ASIDs without flushing.
1014	*/
1015	bool fNewAsid = false;
1016	Assert(pCpu->idCpu != NIL_RTCPUID);
1017	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1018	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes
1019	#ifdef VBOX_WITH_NESTED_HWVIRT
1020	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
1021	#endif
1022	)
1023	{
1024	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1025	pVCpu->hm.s.fForceTLBFlush = true;
1026	fNewAsid = true;
1027	}
1028
1029	/* Set TLB flush state as checked until we return from the world switch. */
1030	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1031
1032	/* Check for explicit TLB flushes. */
1033	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1034	{
1035	pVCpu->hm.s.fForceTLBFlush = true;
1036	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1037	}
1038
1039	/*
1040	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1041	* This Host CPU requirement takes precedence.
1042	*/
1043	if (pVM->hm.s.svm.fAlwaysFlushTLB)
1044	{
1045	pCpu->uCurrentAsid = 1;
1046	pVCpu->hm.s.uCurrentAsid = 1;
1047	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1048	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1049
1050	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1051	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1052
1053	/* Keep track of last CPU ID even when flushing all the time. */
1054	if (fNewAsid)
1055	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1056	}
1057	else
1058	{
1059	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1060	if (pVCpu->hm.s.fForceTLBFlush)
1061	{
1062	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1063	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1064
1065	if (fNewAsid)
1066	{
1067	++pCpu->uCurrentAsid;
1068
1069	bool fHitASIDLimit = false;
1070	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1071	{
1072	pCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1073	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1074	fHitASIDLimit = true;
1075	}
1076
1077	if ( fHitASIDLimit
1078	\|\| pCpu->fFlushAsidBeforeUse)
1079	{
1080	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1081	pCpu->fFlushAsidBeforeUse = false;
1082	}
1083
1084	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1085	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1086	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1087	}
1088	else
1089	{
1090	if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1091	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1092	else
1093	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1094	}
1095
1096	pVCpu->hm.s.fForceTLBFlush = false;
1097	}
1098	}
1099
1100	/* Update VMCB with the ASID. */
1101	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
1102	{
1103	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
1104	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1105	}
1106
1107	AssertMsg(pVCpu->hm.s.idLastCpu == pCpu->idCpu,
1108	("vcpu idLastCpu=%u pcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pCpu->idCpu));
1109	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1110	("Flush count mismatch for cpu %u (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1111	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1112	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
1113	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1114	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1115
1116	#ifdef VBOX_WITH_STATISTICS
1117	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1118	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1119	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1120	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1121	{
1122	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1123	}
1124	else
1125	{
1126	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1127	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1128	}
1129	#endif
1130	}
1131
1132
1133	/** @name 64-bit guest on 32-bit host OS helper functions.
1134	*
1135	* The host CPU is still 64-bit capable but the host OS is running in 32-bit
1136	* mode (code segment, paging). These wrappers/helpers perform the necessary
1137	* bits for the 32->64 switcher.
1138	*
1139	* @{ */
1140	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1141	/**
1142	* Prepares for and executes VMRUN (64-bit guests on a 32-bit host).
1143	*
1144	* @returns VBox status code.
1145	* @param HCPhysVmcbHost Physical address of host VMCB.
1146	* @param HCPhysVmcb Physical address of the VMCB.
1147	* @param pCtx Pointer to the guest-CPU context.
1148	* @param pVM The cross context VM structure.
1149	* @param pVCpu The cross context virtual CPU structure.
1150	*/
1151	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1152	{
1153	uint32_t aParam[8];
1154	aParam[0] = RT_LO_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Lo. */
1155	aParam[1] = RT_HI_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Hi. */
1156	aParam[2] = RT_LO_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Lo. */
1157	aParam[3] = RT_HI_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Hi. */
1158	aParam[4] = VM_RC_ADDR(pVM, pVM);
1159	aParam[5] = 0;
1160	aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1161	aParam[7] = 0;
1162
1163	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1164	}
1165
1166
1167	/**
1168	* Executes the specified VMRUN handler in 64-bit mode.
1169	*
1170	* @returns VBox status code.
1171	* @param pVM The cross context VM structure.
1172	* @param pVCpu The cross context virtual CPU structure.
1173	* @param pCtx Pointer to the guest-CPU context.
1174	* @param enmOp The operation to perform.
1175	* @param cParams Number of parameters.
1176	* @param paParam Array of 32-bit parameters.
1177	*/
1178	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
1179	uint32_t cParams, uint32_t *paParam)
1180	{
1181	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
1182	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
1183
1184	NOREF(pCtx);
1185
1186	/* Disable interrupts. */
1187	RTHCUINTREG uOldEFlags = ASMIntDisableFlags();
1188
1189	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1190	RTCPUID idHostCpu = RTMpCpuId();
1191	CPUMR0SetLApic(pVCpu, idHostCpu);
1192	#endif
1193
1194	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1195	CPUMSetHyperEIP(pVCpu, enmOp);
1196	for (int i = (int)cParams - 1; i >= 0; i--)
1197	CPUMPushHyper(pVCpu, paParam[i]);
1198
1199	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1200	/* Call the switcher. */
1201	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
1202	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1203
1204	/* Restore interrupts. */
1205	ASMSetFlags(uOldEFlags);
1206	return rc;
1207	}
1208
1209	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
1210	/** @} */
1211
1212
1213	/**
1214	* Adds an exception to the intercept exception bitmap in the VMCB and updates
1215	* the corresponding VMCB Clean bit.
1216	*
1217	* @param pVmcb Pointer to the VM control block.
1218	* @param u32Xcpt The value of the exception (X86_XCPT_*).
1219	*/
1220	DECLINLINE(void) hmR0SvmAddXcptIntercept(PSVMVMCB pVmcb, uint32_t u32Xcpt)
1221	{
1222	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(u32Xcpt)))
1223	{
1224	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(u32Xcpt);
1225	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1226	}
1227	}
1228
1229
1230	/**
1231	* Removes an exception from the intercept-exception bitmap in the VMCB and
1232	* updates the corresponding VMCB Clean bit.
1233	*
1234	* @param pVCpu The cross context virtual CPU structure.
1235	* @param pCtx Pointer to the guest-CPU context.
1236	* @param pVmcb Pointer to the VM control block.
1237	* @param u32Xcpt The value of the exception (X86_XCPT_*).
1238	*
1239	* @remarks This takes into account if we're executing a nested-guest and only
1240	* removes the exception intercept if both the guest -and- nested-guest
1241	* are not intercepting it.
1242	*/
1243	DECLINLINE(void) hmR0SvmRemoveXcptIntercept(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb, uint32_t u32Xcpt)
1244	{
1245	Assert(u32Xcpt != X86_XCPT_DB);
1246	Assert(u32Xcpt != X86_XCPT_AC);
1247	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1248	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(u32Xcpt))
1249	{
1250	bool fRemoveXcpt = true;
1251	#ifdef VBOX_WITH_NESTED_HWVIRT
1252	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1253	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1254	{
1255	Assert(pCtx->hwvirt.svm.fHMCachedVmcb); NOREF(pCtx);
1256	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
1257	fRemoveXcpt = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(u32Xcpt));
1258	}
1259	#else
1260	RT_NOREF2(pVCpu, pCtx);
1261	#endif
1262	if (fRemoveXcpt)
1263	{
1264	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(u32Xcpt);
1265	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1266	}
1267	}
1268	#else
1269	RT_NOREF3(pVCpu, pCtx, pVmcb);
1270	#endif
1271	}
1272
1273
1274	/**
1275	* Loads the guest (or nested-guest) CR0 control register into the guest-state
1276	* area in the VMCB.
1277	*
1278	* Although the guest CR0 is a separate field in the VMCB we have to consider
1279	* the FPU state itself which is shared between the host and the guest.
1280	*
1281	* @returns VBox status code.
1282	* @param pVCpu The cross context virtual CPU structure.
1283	* @param pVmcb Pointer to the VM control block.
1284	* @param pCtx Pointer to the guest-CPU context.
1285	*
1286	* @remarks No-long-jump zone!!!
1287	*/
1288	static void hmR0SvmLoadSharedCR0(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1289	{
1290	uint64_t u64GuestCR0 = pCtx->cr0;
1291
1292	/* Always enable caching. */
1293	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1294
1295	/*
1296	* When Nested Paging is not available use shadow page tables and intercept #PFs (the latter done in SVMR0SetupVM()).
1297	*/
1298	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1299	{
1300	u64GuestCR0 \|= X86_CR0_PG; /* When Nested Paging is not available, use shadow page tables. */
1301	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1302	}
1303
1304	/*
1305	* Guest FPU bits.
1306	*/
1307	bool fInterceptNM = false;
1308	bool fInterceptMF = false;
1309	u64GuestCR0 \|= X86_CR0_NE; /* Use internal x87 FPU exceptions handling rather than external interrupts. */
1310	if (CPUMIsGuestFPUStateActive(pVCpu))
1311	{
1312	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1313	if (!(pCtx->cr0 & X86_CR0_NE))
1314	{
1315	Log4(("hmR0SvmLoadSharedCR0: Intercepting Guest CR0.MP Old-style FPU handling!!!\n"));
1316	fInterceptMF = true;
1317	}
1318	}
1319	else
1320	{
1321	fInterceptNM = true; /* Guest FPU inactive, #VMEXIT on #NM for lazy FPU loading. */
1322	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
1323	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
1324	}
1325
1326	/*
1327	* Update the exception intercept bitmap.
1328	*/
1329	if (fInterceptNM)
1330	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_NM);
1331	else
1332	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_NM);
1333
1334	if (fInterceptMF)
1335	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_MF);
1336	else
1337	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_MF);
1338
1339	pVmcb->guest.u64CR0 = u64GuestCR0;
1340	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1341	}
1342
1343
1344	/**
1345	* Loads the guest/nested-guest control registers (CR2, CR3, CR4) into the VMCB.
1346	*
1347	* @returns VBox status code.
1348	* @param pVCpu The cross context virtual CPU structure.
1349	* @param pVmcb Pointer to the VM control block.
1350	* @param pCtx Pointer to the guest-CPU context.
1351	*
1352	* @remarks No-long-jump zone!!!
1353	*/
1354	static int hmR0SvmLoadGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1355	{
1356	PVM pVM = pVCpu->CTX_SUFF(pVM);
1357
1358	/*
1359	* Guest CR2.
1360	*/
1361	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR2))
1362	{
1363	pVmcb->guest.u64CR2 = pCtx->cr2;
1364	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1365	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
1366	}
1367
1368	/*
1369	* Guest CR3.
1370	*/
1371	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
1372	{
1373	if (pVM->hm.s.fNestedPaging)
1374	{
1375	PGMMODE enmShwPagingMode;
1376	#if HC_ARCH_BITS == 32
1377	if (CPUMIsGuestInLongModeEx(pCtx))
1378	enmShwPagingMode = PGMMODE_AMD64_NX;
1379	else
1380	#endif
1381	enmShwPagingMode = PGMGetHostMode(pVM);
1382
1383	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1384	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1385	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1386	pVmcb->guest.u64CR3 = pCtx->cr3;
1387	}
1388	else
1389	{
1390	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1391	Log4(("hmR0SvmLoadGuestControlRegs: CR3=%#RX64 (HyperCR3=%#RX64)\n", pCtx->cr3, pVmcb->guest.u64CR3));
1392	}
1393
1394	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1395	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
1396	}
1397
1398	/*
1399	* Guest CR4.
1400	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
1401	*/
1402	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
1403	{
1404	uint64_t u64GuestCR4 = pCtx->cr4;
1405	Assert(RT_HI_U32(u64GuestCR4) == 0);
1406	if (!pVM->hm.s.fNestedPaging)
1407	{
1408	switch (pVCpu->hm.s.enmShadowMode)
1409	{
1410	case PGMMODE_REAL:
1411	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1412	AssertFailed();
1413	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1414
1415	case PGMMODE_32_BIT: /* 32-bit paging. */
1416	u64GuestCR4 &= ~X86_CR4_PAE;
1417	break;
1418
1419	case PGMMODE_PAE: /* PAE paging. */
1420	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1421	/** Must use PAE paging as we could use physical memory > 4 GB */
1422	u64GuestCR4 \|= X86_CR4_PAE;
1423	break;
1424
1425	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1426	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1427	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1428	break;
1429	#else
1430	AssertFailed();
1431	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1432	#endif
1433
1434	default: /* shut up gcc */
1435	AssertFailed();
1436	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1437	}
1438	}
1439
1440	pVmcb->guest.u64CR4 = u64GuestCR4;
1441	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1442
1443	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1444	pVCpu->hm.s.fLoadSaveGuestXcr0 = (u64GuestCR4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1445
1446	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
1447	}
1448
1449	return VINF_SUCCESS;
1450	}
1451
1452
1453	/**
1454	* Loads the guest segment registers into the VMCB.
1455	*
1456	* @returns VBox status code.
1457	* @param pVCpu The cross context virtual CPU structure.
1458	* @param pVmcb Pointer to the VM control block.
1459	* @param pCtx Pointer to the guest-CPU context.
1460	*
1461	* @remarks No-long-jump zone!!!
1462	*/
1463	static void hmR0SvmLoadGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1464	{
1465	/* Guest Segment registers: CS, SS, DS, ES, FS, GS. */
1466	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
1467	{
1468	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1469	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1470	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1471	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1472	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1473	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1474
1475	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1476	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1477	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
1478	}
1479
1480	/* Guest TR. */
1481	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
1482	{
1483	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1484	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
1485	}
1486
1487	/* Guest LDTR. */
1488	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
1489	{
1490	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1491	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
1492	}
1493
1494	/* Guest GDTR. */
1495	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
1496	{
1497	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1498	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1499	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1500	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
1501	}
1502
1503	/* Guest IDTR. */
1504	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
1505	{
1506	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1507	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1508	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1509	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
1510	}
1511	}
1512
1513
1514	/**
1515	* Loads the guest MSRs into the VMCB.
1516	*
1517	* @param pVCpu The cross context virtual CPU structure.
1518	* @param pVmcb Pointer to the VM control block.
1519	* @param pCtx Pointer to the guest-CPU context.
1520	*
1521	* @remarks No-long-jump zone!!!
1522	*/
1523	static void hmR0SvmLoadGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1524	{
1525	/* Guest Sysenter MSRs. */
1526	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1527	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1528	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1529
1530	/*
1531	* Guest EFER MSR.
1532	* AMD-V requires guest EFER.SVME to be set. Weird.
1533	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1534	*/
1535	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
1536	{
1537	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1538	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1539	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
1540	}
1541
1542	/* 64-bit MSRs. */
1543	if (CPUMIsGuestInLongModeEx(pCtx))
1544	{
1545	pVmcb->guest.FS.u64Base = pCtx->fs.u64Base;
1546	pVmcb->guest.GS.u64Base = pCtx->gs.u64Base;
1547	}
1548	else
1549	{
1550	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit from guest EFER otherwise AMD-V expects amd64 shadow paging. */
1551	if (pCtx->msrEFER & MSR_K6_EFER_LME)
1552	{
1553	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1554	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1555	}
1556	}
1557
1558	/** @todo The following are used in 64-bit only (SYSCALL/SYSRET) but they might
1559	* be writable in 32-bit mode. Clarify with AMD spec. */
1560	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1561	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1562	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1563	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1564	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1565	}
1566
1567
1568	/**
1569	* Loads the guest (or nested-guest) debug state into the VMCB and programs the
1570	* necessary intercepts accordingly.
1571	*
1572	* @param pVCpu The cross context virtual CPU structure.
1573	* @param pVmcb Pointer to the VM control block.
1574	* @param pCtx Pointer to the guest-CPU context.
1575	*
1576	* @remarks No-long-jump zone!!!
1577	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1578	*/
1579	static void hmR0SvmLoadSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1580	{
1581	bool fInterceptMovDRx = false;
1582
1583	/*
1584	* Anyone single stepping on the host side? If so, we'll have to use the
1585	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1586	* the VMM level like the VT-x implementations does.
1587	*/
1588	bool const fStepping = pVCpu->hm.s.fSingleInstruction;
1589	if (fStepping)
1590	{
1591	pVCpu->hm.s.fClearTrapFlag = true;
1592	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1593	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1594	}
1595	else
1596	Assert(!DBGFIsStepping(pVCpu));
1597
1598	if ( fStepping
1599	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1600	{
1601	/*
1602	* Use the combined guest and host DRx values found in the hypervisor
1603	* register set because the debugger has breakpoints active or someone
1604	* is single stepping on the host side.
1605	*
1606	* Note! DBGF expects a clean DR6 state before executing guest code.
1607	*/
1608	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1609	if ( CPUMIsGuestInLongModeEx(pCtx)
1610	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
1611	{
1612	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1613	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1614	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1615	}
1616	else
1617	#endif
1618	if (!CPUMIsHyperDebugStateActive(pVCpu))
1619	{
1620	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1621	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1622	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1623	}
1624
1625	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1626	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1627	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1628	{
1629	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1630	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1631	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1632	pVCpu->hm.s.fUsingHyperDR7 = true;
1633	}
1634
1635	/** @todo If we cared, we could optimize to allow the guest to read registers
1636	* with the same values. */
1637	fInterceptMovDRx = true;
1638	Log5(("hmR0SvmLoadSharedDebugState: Loaded hyper DRx\n"));
1639	}
1640	else
1641	{
1642	/*
1643	* Update DR6, DR7 with the guest values if necessary.
1644	*/
1645	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1646	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1647	{
1648	pVmcb->guest.u64DR7 = pCtx->dr[7];
1649	pVmcb->guest.u64DR6 = pCtx->dr[6];
1650	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1651	pVCpu->hm.s.fUsingHyperDR7 = false;
1652	}
1653
1654	/*
1655	* If the guest has enabled debug registers, we need to load them prior to
1656	* executing guest code so they'll trigger at the right time.
1657	*/
1658	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1659	{
1660	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1661	if ( CPUMIsGuestInLongModeEx(pCtx)
1662	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
1663	{
1664	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1665	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1666	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
1667	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
1668	}
1669	else
1670	#endif
1671	if (!CPUMIsGuestDebugStateActive(pVCpu))
1672	{
1673	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1674	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1675	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1676	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1677	}
1678	Log5(("hmR0SvmLoadSharedDebugState: Loaded guest DRx\n"));
1679	}
1680	/*
1681	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
1682	* intercept #DB as DR6 is updated in the VMCB.
1683	*
1684	* Note! If we cared and dared, we could skip intercepting \#DB here.
1685	* However, \#DB shouldn't be performance critical, so we'll play safe
1686	* and keep the code similar to the VT-x code and always intercept it.
1687	*/
1688	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1689	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
1690	&& !CPUMIsGuestDebugStateActive(pVCpu))
1691	#else
1692	else if (!CPUMIsGuestDebugStateActive(pVCpu))
1693	#endif
1694	{
1695	fInterceptMovDRx = true;
1696	}
1697	}
1698
1699	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
1700	if (fInterceptMovDRx)
1701	{
1702	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1703	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1704	{
1705	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
1706	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
1707	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1708	}
1709	}
1710	else
1711	{
1712	if ( pVmcb->ctrl.u16InterceptRdDRx
1713	\|\| pVmcb->ctrl.u16InterceptWrDRx)
1714	{
1715	pVmcb->ctrl.u16InterceptRdDRx = 0;
1716	pVmcb->ctrl.u16InterceptWrDRx = 0;
1717	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1718	}
1719	}
1720	Log4(("hmR0SvmLoadSharedDebugState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
1721	}
1722
1723
1724	#ifdef VBOX_WITH_NESTED_HWVIRT
1725	/**
1726	* Loads the nested-guest APIC state (currently just the TPR).
1727	*
1728	* @param pVCpu The cross context virtual CPU structure.
1729	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
1730	*/
1731	static void hmR0SvmLoadGuestApicStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst)
1732	{
1733	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1734	{
1735	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
1736	pVmcbNstGst->ctrl.IntCtrl.n.u1VIntrMasking = 1;
1737	pVCpu->hm.s.svm.fSyncVTpr = false;
1738	pVmcbNstGst->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_TPR;
1739
1740	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1741	}
1742	}
1743	#endif
1744
1745	/**
1746	* Loads the guest APIC state (currently just the TPR).
1747	*
1748	* @returns VBox status code.
1749	* @param pVCpu The cross context virtual CPU structure.
1750	* @param pVmcb Pointer to the VM control block.
1751	* @param pCtx Pointer to the guest-CPU context.
1752	*/
1753	static int hmR0SvmLoadGuestApicState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1754	{
1755	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1756	return VINF_SUCCESS;
1757
1758	int rc = VINF_SUCCESS;
1759	PVM pVM = pVCpu->CTX_SUFF(pVM);
1760	if ( PDMHasApic(pVM)
1761	&& APICIsEnabled(pVCpu))
1762	{
1763	bool fPendingIntr;
1764	uint8_t u8Tpr;
1765	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
1766	AssertRCReturn(rc, rc);
1767
1768	/* Assume that we need to trap all TPR accesses and thus need not check on
1769	every #VMEXIT if we should update the TPR. */
1770	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
1771	pVCpu->hm.s.svm.fSyncVTpr = false;
1772
1773	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
1774	if (pVM->hm.s.fTPRPatchingActive)
1775	{
1776	pCtx->msrLSTAR = u8Tpr;
1777	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
1778
1779	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
1780	if (fPendingIntr)
1781	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
1782	else
1783	{
1784	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1785	pVCpu->hm.s.svm.fSyncVTpr = true;
1786	}
1787	}
1788	else
1789	{
1790	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
1791	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
1792
1793	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we can deliver the interrupt to the guest. */
1794	if (fPendingIntr)
1795	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
1796	else
1797	{
1798	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
1799	pVCpu->hm.s.svm.fSyncVTpr = true;
1800	}
1801
1802	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_TPR);
1803	}
1804	}
1805
1806	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1807	return rc;
1808	}
1809
1810
1811	/**
1812	* Loads the exception interrupts required for guest (or nested-guest) execution in
1813	* the VMCB.
1814	*
1815	* @param pVCpu The cross context virtual CPU structure.
1816	* @param pVmcb Pointer to the VM control block.
1817	* @param pCtx Pointer to the guest-CPU context.
1818	*/
1819	static void hmR0SvmLoadGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1820	{
1821	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1822	{
1823	/* Trap #UD for GIM provider (e.g. for hypercalls). */
1824	if (pVCpu->hm.s.fGIMTrapXcptUD)
1825	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_UD);
1826	else
1827	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_UD);
1828
1829	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
1830	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
1831	hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_BP);
1832	else
1833	hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_BP);
1834
1835	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmLoadSharedCR0(). */
1836	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
1837	}
1838	}
1839
1840
1841	#ifdef VBOX_WITH_NESTED_HWVIRT
1842	/**
1843	* Loads the intercepts required for nested-guest execution in the VMCB.
1844	*
1845	* This merges the guest and nested-guest intercepts in a way that if the outer
1846	* guest intercepts an exception we need to intercept it in the nested-guest as
1847	* well and handle it accordingly.
1848	*
1849	* @param pVCpu The cross context virtual CPU structure.
1850	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
1851	* @param pCtx Pointer to the guest-CPU context.
1852	*/
1853	static void hmR0SvmLoadGuestXcptInterceptsNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst, PCPUMCTX pCtx)
1854	{
1855	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1856	{
1857	/* First, load the guest intercepts into the guest VMCB. */
1858	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
1859	Assert(!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR));
1860	hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
1861
1862	/* Next, merge the intercepts into the nested-guest VMCB. */
1863	pVmcbNstGst->ctrl.u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
1864	pVmcbNstGst->ctrl.u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
1865
1866	/* Always intercept CR0, CR4 reads and writes as we alter them. */
1867	pVmcbNstGst->ctrl.u16InterceptRdCRx \|= RT_BIT(0) \| RT_BIT(4);
1868	pVmcbNstGst->ctrl.u16InterceptWrCRx \|= RT_BIT(0) \| RT_BIT(4);
1869
1870	/* Always intercept CR3 reads and writes without nested-paging as we load shadow page tables. */
1871	if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1872	{
1873	pVmcbNstGst->ctrl.u16InterceptRdCRx \|= RT_BIT(3);
1874	pVmcbNstGst->ctrl.u16InterceptWrCRx \|= RT_BIT(3);
1875	}
1876
1877	/** @todo Figure out debugging with nested-guests, till then just intercept
1878	* all DR[0-15] accesses. */
1879	pVmcbNstGst->ctrl.u16InterceptRdDRx \|= 0xffff;
1880	pVmcbNstGst->ctrl.u16InterceptWrDRx \|= 0xffff;
1881
1882	pVmcbNstGst->ctrl.u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
1883	pVmcbNstGst->ctrl.u64InterceptCtrl \|= pVmcb->ctrl.u64InterceptCtrl
1884	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
1885
1886	/*
1887	* Remove control intercepts that we don't need while executing the nested-guest.
1888	*
1889	* VMMCALL when not intercepted raises a \#UD exception in the guest. However,
1890	* other SVM instructions like VMSAVE when not intercept can cause havoc on the
1891	* host as they can write to any location in physical memory, hence they always
1892	* need to be intercepted (they are included in HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS).
1893	*/
1894	Assert( (pVmcbNstGst->ctrl.u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
1895	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
1896	pVmcbNstGst->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_VMMCALL;
1897
1898	/* Finally, update the VMCB clean bits. */
1899	pVmcbNstGst->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1900
1901	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS));
1902	}
1903	}
1904	#endif
1905
1906
1907	/**
1908	* Sets up the appropriate function to run guest code.
1909	*
1910	* @returns VBox status code.
1911	* @param pVCpu The cross context virtual CPU structure.
1912	*
1913	* @remarks No-long-jump zone!!!
1914	*/
1915	static int hmR0SvmSetupVMRunHandler(PVMCPU pVCpu)
1916	{
1917	if (CPUMIsGuestInLongMode(pVCpu))
1918	{
1919	#ifndef VBOX_ENABLE_64_BITS_GUESTS
1920	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1921	#endif
1922	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
1923	#if HC_ARCH_BITS == 32
1924	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
1925	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
1926	#else
1927	/* 64-bit host or hybrid host. */
1928	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
1929	#endif
1930	}
1931	else
1932	{
1933	/* Guest is not in long mode, use the 32-bit handler. */
1934	pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
1935	}
1936	return VINF_SUCCESS;
1937	}
1938
1939
1940	/**
1941	* Enters the AMD-V session.
1942	*
1943	* @returns VBox status code.
1944	* @param pVM The cross context VM structure.
1945	* @param pVCpu The cross context virtual CPU structure.
1946	* @param pCpu Pointer to the CPU info struct.
1947	*/
1948	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1949	{
1950	AssertPtr(pVM);
1951	AssertPtr(pVCpu);
1952	Assert(pVM->hm.s.svm.fSupported);
1953	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1954	NOREF(pVM); NOREF(pCpu);
1955
1956	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1957	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
1958
1959	pVCpu->hm.s.fLeaveDone = false;
1960	return VINF_SUCCESS;
1961	}
1962
1963
1964	/**
1965	* Thread-context callback for AMD-V.
1966	*
1967	* @param enmEvent The thread-context event.
1968	* @param pVCpu The cross context virtual CPU structure.
1969	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
1970	* @thread EMT(pVCpu)
1971	*/
1972	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
1973	{
1974	NOREF(fGlobalInit);
1975
1976	switch (enmEvent)
1977	{
1978	case RTTHREADCTXEVENT_OUT:
1979	{
1980	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1981	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
1982	VMCPU_ASSERT_EMT(pVCpu);
1983
1984	/* No longjmps (log-flush, locks) in this fragile context. */
1985	VMMRZCallRing3Disable(pVCpu);
1986
1987	if (!pVCpu->hm.s.fLeaveDone)
1988	{
1989	hmR0SvmLeave(pVCpu);
1990	pVCpu->hm.s.fLeaveDone = true;
1991	}
1992
1993	/* Leave HM context, takes care of local init (term). */
1994	int rc = HMR0LeaveCpu(pVCpu);
1995	AssertRC(rc); NOREF(rc);
1996
1997	/* Restore longjmp state. */
1998	VMMRZCallRing3Enable(pVCpu);
1999	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2000	break;
2001	}
2002
2003	case RTTHREADCTXEVENT_IN:
2004	{
2005	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2006	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2007	VMCPU_ASSERT_EMT(pVCpu);
2008
2009	/* No longjmps (log-flush, locks) in this fragile context. */
2010	VMMRZCallRing3Disable(pVCpu);
2011
2012	/*
2013	* Initialize the bare minimum state required for HM. This takes care of
2014	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2015	*/
2016	int rc = HMR0EnterCpu(pVCpu);
2017	AssertRC(rc); NOREF(rc);
2018	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
2019
2020	pVCpu->hm.s.fLeaveDone = false;
2021
2022	/* Restore longjmp state. */
2023	VMMRZCallRing3Enable(pVCpu);
2024	break;
2025	}
2026
2027	default:
2028	break;
2029	}
2030	}
2031
2032
2033	/**
2034	* Saves the host state.
2035	*
2036	* @returns VBox status code.
2037	* @param pVM The cross context VM structure.
2038	* @param pVCpu The cross context virtual CPU structure.
2039	*
2040	* @remarks No-long-jump zone!!!
2041	*/
2042	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
2043	{
2044	NOREF(pVM);
2045	NOREF(pVCpu);
2046	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2047	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
2048	return VINF_SUCCESS;
2049	}
2050
2051
2052	/**
2053	* Loads the guest state into the VMCB.
2054	*
2055	* The CPU state will be loaded from these fields on every successful VM-entry.
2056	* Also sets up the appropriate VMRUN function to execute guest code based on
2057	* the guest CPU mode.
2058	*
2059	* @returns VBox status code.
2060	* @param pVM The cross context VM structure.
2061	* @param pVCpu The cross context virtual CPU structure.
2062	* @param pCtx Pointer to the guest-CPU context.
2063	*
2064	* @remarks No-long-jump zone!!!
2065	*/
2066	static int hmR0SvmLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2067	{
2068	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
2069
2070	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2071	AssertMsgReturn(pVmcb, ("Invalid pVmcb\n"), VERR_SVM_INVALID_PVMCB);
2072
2073	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2074
2075	int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcb, pCtx);
2076	AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestControlRegs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2077
2078	hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcb, pCtx);
2079	hmR0SvmLoadGuestMsrs(pVCpu, pVmcb, pCtx);
2080
2081	pVmcb->guest.u64RIP = pCtx->rip;
2082	pVmcb->guest.u64RSP = pCtx->rsp;
2083	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2084	pVmcb->guest.u64RAX = pCtx->rax;
2085
2086	rc = hmR0SvmLoadGuestApicState(pVCpu, pVmcb, pCtx);
2087	AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2088
2089	hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
2090
2091	rc = hmR0SvmSetupVMRunHandler(pVCpu);
2092	AssertLogRelMsgRCReturn(rc, ("hmR0SvmSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2093
2094	/* Clear any unused and reserved bits. */
2095	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2096	\| HM_CHANGED_GUEST_RSP
2097	\| HM_CHANGED_GUEST_RFLAGS
2098	\| HM_CHANGED_GUEST_SYSENTER_CS_MSR
2099	\| HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2100	\| HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2101	\| HM_CHANGED_GUEST_LAZY_MSRS /* Unused. */
2102	\| HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2103	\| HM_CHANGED_SVM_RESERVED2
2104	\| HM_CHANGED_SVM_RESERVED3
2105	\| HM_CHANGED_SVM_RESERVED4);
2106
2107	/* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2108	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2109	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
2110	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2111
2112	Log4(("hmR0SvmLoadGuestState: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 CR4=%#RX32\n", pCtx->cs.Sel, pCtx->rip,
2113	pCtx->eflags.u, pCtx->cr0, pCtx->cr3, pCtx->cr4));
2114	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2115	return rc;
2116	}
2117
2118
2119	#ifdef VBOX_WITH_NESTED_HWVIRT
2120	/**
2121	* Caches the nested-guest VMCB fields before we modify them for execution using
2122	* hardware-assisted SVM.
2123	*
2124	* @returns true if the VMCB was previously already cached, false otherwise.
2125	* @param pCtx Pointer to the guest-CPU context.
2126	*
2127	* @sa HMSvmNstGstVmExitNotify.
2128	*/
2129	static bool hmR0SvmVmRunCacheVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
2130	{
2131	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2132	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2133	PCSVMVMCBSTATESAVE pVmcbNstGstState = &pVmcbNstGst->guest;
2134	PSVMNESTEDVMCBCACHE pNstGstVmcbCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2135
2136	/*
2137	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2138	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2139	*
2140	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2141	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2142	*/
2143	bool const fWasCached = pCtx->hwvirt.svm.fHMCachedVmcb;
2144	if (!fWasCached)
2145	{
2146	pNstGstVmcbCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2147	pNstGstVmcbCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2148	pNstGstVmcbCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2149	pNstGstVmcbCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2150	pNstGstVmcbCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2151	pNstGstVmcbCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2152	pNstGstVmcbCache->u64CR0 = pVmcbNstGstState->u64CR0;
2153	pNstGstVmcbCache->u64CR3 = pVmcbNstGstState->u64CR3;
2154	pNstGstVmcbCache->u64CR4 = pVmcbNstGstState->u64CR4;
2155	pNstGstVmcbCache->u64EFER = pVmcbNstGstState->u64EFER;
2156	pNstGstVmcbCache->u64IOPMPhysAddr = pVmcbNstGstCtrl->u64IOPMPhysAddr;
2157	pNstGstVmcbCache->u64MSRPMPhysAddr = pVmcbNstGstCtrl->u64MSRPMPhysAddr;
2158	pNstGstVmcbCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2159	pNstGstVmcbCache->u32VmcbCleanBits = pVmcbNstGstCtrl->u32VmcbCleanBits;
2160	pNstGstVmcbCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2161	pNstGstVmcbCache->TLBCtrl = pVmcbNstGstCtrl->TLBCtrl;
2162	pNstGstVmcbCache->u1NestedPaging = pVmcbNstGstCtrl->u1NestedPaging;
2163	pCtx->hwvirt.svm.fHMCachedVmcb = true;
2164	Log4(("hmR0SvmVmRunCacheVmcb: Cached VMCB fields\n"));
2165	}
2166
2167	return fWasCached;
2168	}
2169
2170
2171	/**
2172	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2173	*
2174	* @param pVCpu The cross context virtual CPU structure.
2175	* @param pCtx Pointer to the guest-CPU context.
2176	*/
2177	static void hmR0SvmVmRunSetupVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
2178	{
2179	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2180	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2181
2182	/*
2183	* First cache the nested-guest VMCB fields we may potentially modify.
2184	*/
2185	bool const fVmcbCached = hmR0SvmVmRunCacheVmcb(pVCpu, pCtx);
2186	if (!fVmcbCached)
2187	{
2188	/*
2189	* The IOPM of the nested-guest can be ignored because the the guest always
2190	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2191	* into the nested-guest one and swap it back on the #VMEXIT.
2192	*/
2193	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2194
2195	/*
2196	* Load the host-physical address into the MSRPM rather than the nested-guest
2197	* physical address (currently we trap all MSRs in the nested-guest).
2198	*/
2199	pVmcbNstGstCtrl->u64MSRPMPhysAddr = g_HCPhysNstGstMsrBitmap;
2200
2201	/*
2202	* Use the same nested-paging as the "outer" guest. We can't dynamically
2203	* switch off nested-paging suddenly while executing a VM (see assertion at the
2204	* end of Trap0eHandler in PGMAllBth.h).
2205	*/
2206	pVmcbNstGstCtrl->u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
2207	}
2208	else
2209	{
2210	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2211	Assert(pVmcbNstGstCtrl->u64MSRPMPhysAddr = g_HCPhysNstGstMsrBitmap);
2212	Assert(RT_BOOL(pVmcbNstGstCtrl->u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
2213	}
2214	}
2215
2216
2217	/**
2218	* Loads the nested-guest state into the VMCB.
2219	*
2220	* @returns VBox status code.
2221	* @param pVCpu The cross context virtual CPU structure.
2222	* @param pCtx Pointer to the guest-CPU context.
2223	*
2224	* @remarks No-long-jump zone!!!
2225	*/
2226	static int hmR0SvmLoadGuestStateNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2227	{
2228	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2229
2230	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2231	Assert(pVmcbNstGst);
2232
2233	hmR0SvmVmRunSetupVmcb(pVCpu, pCtx);
2234
2235	int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcbNstGst, pCtx);
2236	AssertRCReturn(rc, rc);
2237
2238	hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcbNstGst, pCtx);
2239	hmR0SvmLoadGuestMsrs(pVCpu, pVmcbNstGst, pCtx);
2240	hmR0SvmLoadGuestApicStateNested(pVCpu, pVmcbNstGst);
2241
2242	pVmcbNstGst->guest.u64RIP = pCtx->rip;
2243	pVmcbNstGst->guest.u64RSP = pCtx->rsp;
2244	pVmcbNstGst->guest.u64RFlags = pCtx->eflags.u32;
2245	pVmcbNstGst->guest.u64RAX = pCtx->rax;
2246
2247	hmR0SvmLoadGuestXcptInterceptsNested(pVCpu, pVmcbNstGst, pCtx);
2248
2249	rc = hmR0SvmSetupVMRunHandler(pVCpu);
2250	AssertRCReturn(rc, rc);
2251
2252	/* Clear any unused and reserved bits. */
2253	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2254	\| HM_CHANGED_GUEST_RSP
2255	\| HM_CHANGED_GUEST_RFLAGS
2256	\| HM_CHANGED_GUEST_SYSENTER_CS_MSR
2257	\| HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2258	\| HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2259	\| HM_CHANGED_GUEST_LAZY_MSRS /* Unused. */
2260	\| HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2261	\| HM_CHANGED_SVM_RESERVED2
2262	\| HM_CHANGED_SVM_RESERVED3
2263	\| HM_CHANGED_SVM_RESERVED4);
2264
2265	/* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2266	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2267	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
2268	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2269
2270	Log4(("hmR0SvmLoadGuestStateNested: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 (HyperCR3=%#RX64) CR4=%#RX32 "
2271	"ESP=%#RX32 EBP=%#RX32 rc=%d\n", pCtx->cs.Sel, pCtx->rip, pCtx->eflags.u, pCtx->cr0, pCtx->cr3,
2272	pVmcbNstGst->guest.u64CR3, pCtx->cr4, pCtx->esp, pCtx->ebp, rc));
2273	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2274
2275	return rc;
2276	}
2277	#endif
2278
2279
2280	/**
2281	* Loads the state shared between the host and guest or nested-guest into the
2282	* VMCB.
2283	*
2284	* @param pVCpu The cross context virtual CPU structure.
2285	* @param pVmcb Pointer to the VM control block.
2286	* @param pCtx Pointer to the guest-CPU context.
2287	*
2288	* @remarks No-long-jump zone!!!
2289	*/
2290	static void hmR0SvmLoadSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
2291	{
2292	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2293	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2294
2295	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
2296	{
2297	hmR0SvmLoadSharedCR0(pVCpu, pVmcb, pCtx);
2298	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
2299	}
2300
2301	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
2302	{
2303	/** @todo Figure out stepping with nested-guest. */
2304	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2305	hmR0SvmLoadSharedDebugState(pVCpu, pVmcb, pCtx);
2306	else
2307	{
2308	pVmcb->guest.u64DR6 = pCtx->dr[6];
2309	pVmcb->guest.u64DR7 = pCtx->dr[7];
2310	Log4(("hmR0SvmLoadSharedState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2311	}
2312
2313	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
2314	}
2315
2316	/* Unused on AMD-V. */
2317	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
2318
2319	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
2320	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2321	}
2322
2323
2324	/**
2325	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU context.
2326	*
2327	* Currently there is no residual state left in the CPU that is not updated in the
2328	* VMCB.
2329	*
2330	* @returns VBox status code.
2331	* @param pVCpu The cross context virtual CPU structure.
2332	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2333	* out-of-sync. Make sure to update the required fields
2334	* before using them.
2335	* @param pVmcb Pointer to the VM control block.
2336	*/
2337	static void hmR0SvmSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PCSVMVMCB pVmcb)
2338	{
2339	Assert(VMMRZCallRing3IsEnabled(pVCpu));
2340
2341	pMixedCtx->rip = pVmcb->guest.u64RIP;
2342	pMixedCtx->rsp = pVmcb->guest.u64RSP;
2343	pMixedCtx->eflags.u32 = pVmcb->guest.u64RFlags;
2344	pMixedCtx->rax = pVmcb->guest.u64RAX;
2345
2346	/*
2347	* Guest interrupt shadow.
2348	*/
2349	if (pVmcb->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
2350	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
2351	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2352	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2353
2354	/*
2355	* Guest Control registers: CR2, CR3 (handled at the end) - accesses to other control registers are always intercepted.
2356	*/
2357	pMixedCtx->cr2 = pVmcb->guest.u64CR2;
2358
2359	/*
2360	* Guest MSRs.
2361	*/
2362	pMixedCtx->msrSTAR = pVmcb->guest.u64STAR; /* legacy syscall eip, cs & ss */
2363	pMixedCtx->msrLSTAR = pVmcb->guest.u64LSTAR; /* 64-bit mode syscall rip */
2364	pMixedCtx->msrCSTAR = pVmcb->guest.u64CSTAR; /* compatibility mode syscall rip */
2365	pMixedCtx->msrSFMASK = pVmcb->guest.u64SFMASK; /* syscall flag mask */
2366	pMixedCtx->msrKERNELGSBASE = pVmcb->guest.u64KernelGSBase; /* swapgs exchange value */
2367	pMixedCtx->SysEnter.cs = pVmcb->guest.u64SysEnterCS;
2368	pMixedCtx->SysEnter.eip = pVmcb->guest.u64SysEnterEIP;
2369	pMixedCtx->SysEnter.esp = pVmcb->guest.u64SysEnterESP;
2370
2371	/*
2372	* Guest segment registers (includes FS, GS base MSRs for 64-bit guests).
2373	*/
2374	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, CS, cs);
2375	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, SS, ss);
2376	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, DS, ds);
2377	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, ES, es);
2378	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, FS, fs);
2379	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, GS, gs);
2380
2381	/*
2382	* Correct the hidden CS granularity bit. Haven't seen it being wrong in any other
2383	* register (yet).
2384	*/
2385	/** @todo SELM might need to be fixed as it too should not care about the
2386	* granularity bit. See @bugref{6785}. */
2387	if ( !pMixedCtx->cs.Attr.n.u1Granularity
2388	&& pMixedCtx->cs.Attr.n.u1Present
2389	&& pMixedCtx->cs.u32Limit > UINT32_C(0xfffff))
2390	{
2391	Assert((pMixedCtx->cs.u32Limit & 0xfff) == 0xfff);
2392	pMixedCtx->cs.Attr.n.u1Granularity = 1;
2393	}
2394
2395	#ifdef VBOX_STRICT
2396	# define HMSVM_ASSERT_SEG_GRANULARITY(reg) \
2397	AssertMsg( !pMixedCtx->reg.Attr.n.u1Present \
2398	\|\| ( pMixedCtx->reg.Attr.n.u1Granularity \
2399	? (pMixedCtx->reg.u32Limit & 0xfff) == 0xfff \
2400	: pMixedCtx->reg.u32Limit <= UINT32_C(0xfffff)), \
2401	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", pMixedCtx->reg.u32Limit, \
2402	pMixedCtx->reg.Attr.u, pMixedCtx->reg.u64Base))
2403
2404	HMSVM_ASSERT_SEG_GRANULARITY(cs);
2405	HMSVM_ASSERT_SEG_GRANULARITY(ss);
2406	HMSVM_ASSERT_SEG_GRANULARITY(ds);
2407	HMSVM_ASSERT_SEG_GRANULARITY(es);
2408	HMSVM_ASSERT_SEG_GRANULARITY(fs);
2409	HMSVM_ASSERT_SEG_GRANULARITY(gs);
2410
2411	# undef HMSVM_ASSERT_SEL_GRANULARITY
2412	#endif
2413
2414	/*
2415	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the VMCB and uses that
2416	* and thus it's possible that when the CPL changes during guest execution that the SS DPL
2417	* isn't updated by AMD-V. Observed on some AMD Fusion CPUs with 64-bit guests.
2418	* See AMD spec. 15.5.1 "Basic operation".
2419	*/
2420	Assert(!(pVmcb->guest.u8CPL & ~0x3));
2421	pMixedCtx->ss.Attr.n.u2Dpl = pVmcb->guest.u8CPL & 0x3;
2422
2423	/*
2424	* Guest TR.
2425	* Fixup TR attributes so it's compatible with Intel. Important when saved-states are used
2426	* between Intel and AMD. See @bugref{6208#c39}.
2427	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2428	*/
2429	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, TR, tr);
2430	if (pMixedCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2431	{
2432	if ( pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2433	\|\| CPUMIsGuestInLongModeEx(pMixedCtx))
2434	pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2435	else if (pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2436	pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2437	}
2438
2439	/*
2440	* Guest Descriptor-Table registers.
2441	*/
2442	HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, LDTR, ldtr);
2443	pMixedCtx->gdtr.cbGdt = pVmcb->guest.GDTR.u32Limit;
2444	pMixedCtx->gdtr.pGdt = pVmcb->guest.GDTR.u64Base;
2445
2446	pMixedCtx->idtr.cbIdt = pVmcb->guest.IDTR.u32Limit;
2447	pMixedCtx->idtr.pIdt = pVmcb->guest.IDTR.u64Base;
2448
2449	/*
2450	* Guest Debug registers.
2451	*/
2452	if (!pVCpu->hm.s.fUsingHyperDR7)
2453	{
2454	pMixedCtx->dr[6] = pVmcb->guest.u64DR6;
2455	pMixedCtx->dr[7] = pVmcb->guest.u64DR7;
2456	}
2457	else
2458	{
2459	Assert(pVmcb->guest.u64DR7 == CPUMGetHyperDR7(pVCpu));
2460	CPUMSetHyperDR6(pVCpu, pVmcb->guest.u64DR6);
2461	}
2462
2463	/*
2464	* With Nested Paging, CR3 changes are not intercepted. Therefore, sync. it now.
2465	* This is done as the very last step of syncing the guest state, as PGMUpdateCR3() may cause longjmp's to ring-3.
2466	*/
2467	if ( pVmcb->ctrl.u1NestedPaging
2468	&& pMixedCtx->cr3 != pVmcb->guest.u64CR3)
2469	{
2470	CPUMSetGuestCR3(pVCpu, pVmcb->guest.u64CR3);
2471	PGMUpdateCR3(pVCpu, pVmcb->guest.u64CR3);
2472	}
2473
2474	Log4(("hmR0SvmSaveGuestState: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 CR4=%#RX32\n", pMixedCtx->cs.Sel,
2475	pMixedCtx->rip, pMixedCtx->eflags.u, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4));
2476	}
2477
2478
2479	/**
2480	* Does the necessary state syncing before returning to ring-3 for any reason
2481	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2482	*
2483	* @param pVCpu The cross context virtual CPU structure.
2484	*
2485	* @remarks No-long-jmp zone!!!
2486	*/
2487	static void hmR0SvmLeave(PVMCPU pVCpu)
2488	{
2489	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2490	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2491	Assert(VMMR0IsLogFlushDisabled(pVCpu));
2492
2493	/*
2494	* !!! IMPORTANT !!!
2495	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2496	*/
2497
2498	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
2499	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
2500	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0); /** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2501
2502	/*
2503	* Restore host debug registers if necessary and resync on next R0 reentry.
2504	*/
2505	#ifdef VBOX_STRICT
2506	if (CPUMIsHyperDebugStateActive(pVCpu))
2507	{
2508	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
2509	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2510	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2511	}
2512	#endif
2513	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */))
2514	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);/** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2515
2516	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2517	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2518
2519	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2520	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
2521	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2522	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2523	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2524
2525	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
2526	}
2527
2528
2529	/**
2530	* Leaves the AMD-V session.
2531	*
2532	* @returns VBox status code.
2533	* @param pVCpu The cross context virtual CPU structure.
2534	*/
2535	static int hmR0SvmLeaveSession(PVMCPU pVCpu)
2536	{
2537	HM_DISABLE_PREEMPT();
2538	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2539	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2540
2541	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
2542	and done this from the SVMR0ThreadCtxCallback(). */
2543	if (!pVCpu->hm.s.fLeaveDone)
2544	{
2545	hmR0SvmLeave(pVCpu);
2546	pVCpu->hm.s.fLeaveDone = true;
2547	}
2548
2549	/*
2550	* !!! IMPORTANT !!!
2551	* If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2552	*/
2553
2554	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2555	/* Deregister hook now that we've left HM context before re-enabling preemption. */
2556	VMMR0ThreadCtxHookDisable(pVCpu);
2557
2558	/* Leave HM context. This takes care of local init (term). */
2559	int rc = HMR0LeaveCpu(pVCpu);
2560
2561	HM_RESTORE_PREEMPT();
2562	return rc;
2563	}
2564
2565
2566	/**
2567	* Does the necessary state syncing before doing a longjmp to ring-3.
2568	*
2569	* @returns VBox status code.
2570	* @param pVCpu The cross context virtual CPU structure.
2571	*
2572	* @remarks No-long-jmp zone!!!
2573	*/
2574	static int hmR0SvmLongJmpToRing3(PVMCPU pVCpu)
2575	{
2576	return hmR0SvmLeaveSession(pVCpu);
2577	}
2578
2579
2580	/**
2581	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
2582	* any remaining host state) before we longjump to ring-3 and possibly get
2583	* preempted.
2584	*
2585	* @param pVCpu The cross context virtual CPU structure.
2586	* @param enmOperation The operation causing the ring-3 longjump.
2587	* @param pvUser The user argument (pointer to the possibly
2588	* out-of-date guest-CPU context).
2589	*/
2590	static DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
2591	{
2592	RT_NOREF_PV(pvUser);
2593
2594	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
2595	{
2596	/*
2597	* !!! IMPORTANT !!!
2598	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
2599	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
2600	*/
2601	VMMRZCallRing3RemoveNotification(pVCpu);
2602	VMMRZCallRing3Disable(pVCpu);
2603	HM_DISABLE_PREEMPT();
2604
2605	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2606	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2607
2608	/* Restore host debug registers if necessary and resync on next R0 reentry. */
2609	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2610
2611	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
2612	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2613	VMMR0ThreadCtxHookDisable(pVCpu);
2614
2615	/* Leave HM context. This takes care of local init (term). */
2616	HMR0LeaveCpu(pVCpu);
2617
2618	HM_RESTORE_PREEMPT();
2619	return VINF_SUCCESS;
2620	}
2621
2622	Assert(pVCpu);
2623	Assert(pvUser);
2624	Assert(VMMRZCallRing3IsEnabled(pVCpu));
2625	HMSVM_ASSERT_PREEMPT_SAFE();
2626
2627	VMMRZCallRing3Disable(pVCpu);
2628	Assert(VMMR0IsLogFlushDisabled(pVCpu));
2629
2630	Log4(("hmR0SvmCallRing3Callback->hmR0SvmLongJmpToRing3\n"));
2631	int rc = hmR0SvmLongJmpToRing3(pVCpu);
2632	AssertRCReturn(rc, rc);
2633
2634	VMMRZCallRing3Enable(pVCpu);
2635	return VINF_SUCCESS;
2636	}
2637
2638
2639	/**
2640	* Take necessary actions before going back to ring-3.
2641	*
2642	* An action requires us to go back to ring-3. This function does the necessary
2643	* steps before we can safely return to ring-3. This is not the same as longjmps
2644	* to ring-3, this is voluntary.
2645	*
2646	* @returns VBox status code.
2647	* @param pVM The cross context VM structure.
2648	* @param pVCpu The cross context virtual CPU structure.
2649	* @param pCtx Pointer to the guest-CPU context.
2650	* @param rcExit The reason for exiting to ring-3. Can be
2651	* VINF_VMM_UNKNOWN_RING3_CALL.
2652	*/
2653	static int hmR0SvmExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rcExit)
2654	{
2655	Assert(pVM);
2656	Assert(pVCpu);
2657	Assert(pCtx);
2658	HMSVM_ASSERT_PREEMPT_SAFE();
2659
2660	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
2661	VMMRZCallRing3Disable(pVCpu);
2662	Log4(("hmR0SvmExitToRing3: VCPU[%u]: rcExit=%d LocalFF=%#RX32 GlobalFF=%#RX32\n", pVCpu->idCpu, rcExit,
2663	pVCpu->fLocalForcedActions, pVM->fGlobalForcedActions));
2664
2665	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
2666	if (pVCpu->hm.s.Event.fPending)
2667	{
2668	hmR0SvmPendingEventToTrpmTrap(pVCpu);
2669	Assert(!pVCpu->hm.s.Event.fPending);
2670	}
2671
2672	/* Sync. the necessary state for going back to ring-3. */
2673	hmR0SvmLeaveSession(pVCpu);
2674	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2675
2676	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2677	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
2678	\| CPUM_CHANGED_LDTR
2679	\| CPUM_CHANGED_GDTR
2680	\| CPUM_CHANGED_IDTR
2681	\| CPUM_CHANGED_TR
2682	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
2683	if ( pVM->hm.s.fNestedPaging
2684	&& CPUMIsGuestPagingEnabledEx(pCtx))
2685	{
2686	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
2687	}
2688
2689	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
2690	if (rcExit != VINF_EM_RAW_INTERRUPT)
2691	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
2692
2693	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
2694
2695	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
2696	VMMRZCallRing3RemoveNotification(pVCpu);
2697	VMMRZCallRing3Enable(pVCpu);
2698
2699	/*
2700	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
2701	* and if we're injecting an event we should have a TRPM trap pending.
2702	*/
2703	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
2704	pVCpu->hm.s.u32HMError = rcExit,
2705	VERR_SVM_IPE_5);
2706	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
2707	pVCpu->hm.s.u32HMError = rcExit,
2708	VERR_SVM_IPE_4);
2709
2710	return rcExit;
2711	}
2712
2713
2714	#ifdef VBOX_WITH_NESTED_HWVIRT
2715	/**
2716	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
2717	* intercepts for the nested-guest.
2718	*
2719	* @param pVM The cross context VM structure.
2720	* @param pVCpu The cross context virtual CPU structure.
2721	* @param pCtx Pointer to the nested guest-CPU context.
2722	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2723	*
2724	* @remarks No-long-jump zone!!!
2725	*/
2726	static void hmR0SvmUpdateTscOffsettingNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcbNstGst)
2727	{
2728	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
2729
2730	bool fParavirtTsc;
2731	uint64_t uTscOffset;
2732	bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
2733
2734	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2735	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2736
2737	/*
2738	* Only avoid intercepting if we determined the host TSC (++) is stable enough
2739	* to not intercept -and- the nested-hypervisor itself does not want to intercept it.
2740	*/
2741	if ( fCanUseRealTsc
2742	&& !(pVmcbNstGstCache->u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
2743	{
2744	pVmcbNstGstCtrl->u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSC;
2745	pVmcbNstGstCtrl->u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSCP;
2746	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2747	}
2748	else
2749	{
2750	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSC;
2751	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSCP;
2752	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2753	}
2754
2755	/* Apply the nested-guest VMCB's TSC offset over the guest one. */
2756	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
2757
2758	/* Update the nested-guest VMCB with the combined TSC offset (of guest and nested-guest). */
2759	pVmcbNstGstCtrl->u64TSCOffset = uTscOffset;
2760
2761	/* Finally update the VMCB clean bits since we touched the intercepts as well as the TSC offset. */
2762	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2763
2764	if (fParavirtTsc)
2765	{
2766	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
2767	information before every VM-entry, hence disable it for performance sake. */
2768	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2769	}
2770	}
2771	#endif
2772
2773
2774	/**
2775	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
2776	* intercepts.
2777	*
2778	* @param pVM The cross context VM structure.
2779	* @param pVCpu The cross context virtual CPU structure.
2780	* @param pVmcb Pointer to the VM control block.
2781	*
2782	* @remarks No-long-jump zone!!!
2783	*/
2784	static void hmR0SvmUpdateTscOffsetting(PVM pVM, PVMCPU pVCpu, PSVMVMCB pVmcb)
2785	{
2786	bool fParavirtTsc;
2787	bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVmcb->ctrl.u64TSCOffset, &fParavirtTsc);
2788	if (fCanUseRealTsc)
2789	{
2790	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSC;
2791	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSCP;
2792	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2793	}
2794	else
2795	{
2796	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSC;
2797	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_RDTSCP;
2798	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2799	}
2800	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2801
2802	/** @todo later optimize this to be done elsewhere and not before every
2803	* VM-entry. */
2804	if (fParavirtTsc)
2805	{
2806	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
2807	information before every VM-entry, hence disable it for performance sake. */
2808	#if 0
2809	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
2810	AssertRC(rc);
2811	#endif
2812	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2813	}
2814	}
2815
2816
2817	/**
2818	* Sets an event as a pending event to be injected into the guest.
2819	*
2820	* @param pVCpu The cross context virtual CPU structure.
2821	* @param pEvent Pointer to the SVM event.
2822	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
2823	* page-fault.
2824	*
2825	* @remarks Statistics counter assumes this is a guest event being reflected to
2826	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
2827	*/
2828	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
2829	{
2830	Assert(!pVCpu->hm.s.Event.fPending);
2831	Assert(pEvent->n.u1Valid);
2832
2833	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
2834	pVCpu->hm.s.Event.fPending = true;
2835	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
2836
2837	Log4(("hmR0SvmSetPendingEvent: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
2838	pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
2839	}
2840
2841
2842	/**
2843	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
2844	*
2845	* @param pVCpu The cross context virtual CPU structure.
2846	*/
2847	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
2848	{
2849	SVMEVENT Event;
2850	Event.u = 0;
2851	Event.n.u1Valid = 1;
2852	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2853	Event.n.u8Vector = X86_XCPT_UD;
2854	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2855	}
2856
2857
2858	/**
2859	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
2860	*
2861	* @param pVCpu The cross context virtual CPU structure.
2862	*/
2863	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
2864	{
2865	SVMEVENT Event;
2866	Event.u = 0;
2867	Event.n.u1Valid = 1;
2868	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2869	Event.n.u8Vector = X86_XCPT_DB;
2870	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2871	}
2872
2873
2874	/**
2875	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
2876	*
2877	* @param pVCpu The cross context virtual CPU structure.
2878	* @param pCtx Pointer to the guest-CPU context.
2879	* @param u32ErrCode The error-code for the page-fault.
2880	* @param uFaultAddress The page fault address (CR2).
2881	*
2882	* @remarks This updates the guest CR2 with @a uFaultAddress!
2883	*/
2884	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
2885	{
2886	SVMEVENT Event;
2887	Event.u = 0;
2888	Event.n.u1Valid = 1;
2889	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2890	Event.n.u8Vector = X86_XCPT_PF;
2891	Event.n.u1ErrorCodeValid = 1;
2892	Event.n.u32ErrorCode = u32ErrCode;
2893
2894	/* Update CR2 of the guest. */
2895	if (pCtx->cr2 != uFaultAddress)
2896	{
2897	pCtx->cr2 = uFaultAddress;
2898	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
2899	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR2);
2900	}
2901
2902	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
2903	}
2904
2905
2906	/**
2907	* Sets a device-not-available (\#NM) exception as pending-for-injection into
2908	* the VM.
2909	*
2910	* @param pVCpu The cross context virtual CPU structure.
2911	*/
2912	DECLINLINE(void) hmR0SvmSetPendingXcptNM(PVMCPU pVCpu)
2913	{
2914	SVMEVENT Event;
2915	Event.u = 0;
2916	Event.n.u1Valid = 1;
2917	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2918	Event.n.u8Vector = X86_XCPT_NM;
2919	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2920	}
2921
2922
2923	/**
2924	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
2925	*
2926	* @param pVCpu The cross context virtual CPU structure.
2927	*/
2928	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
2929	{
2930	SVMEVENT Event;
2931	Event.u = 0;
2932	Event.n.u1Valid = 1;
2933	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2934	Event.n.u8Vector = X86_XCPT_MF;
2935	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2936	}
2937
2938
2939	/**
2940	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
2941	*
2942	* @param pVCpu The cross context virtual CPU structure.
2943	*/
2944	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
2945	{
2946	SVMEVENT Event;
2947	Event.u = 0;
2948	Event.n.u1Valid = 1;
2949	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2950	Event.n.u8Vector = X86_XCPT_DF;
2951	Event.n.u1ErrorCodeValid = 1;
2952	Event.n.u32ErrorCode = 0;
2953	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2954	}
2955
2956
2957	/**
2958	* Injects an event into the guest upon VMRUN by updating the relevant field
2959	* in the VMCB.
2960	*
2961	* @param pVCpu The cross context virtual CPU structure.
2962	* @param pVmcb Pointer to the guest VM control block.
2963	* @param pCtx Pointer to the guest-CPU context.
2964	* @param pEvent Pointer to the event.
2965	*
2966	* @remarks No-long-jump zone!!!
2967	* @remarks Requires CR0!
2968	*/
2969	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx, PSVMEVENT pEvent)
2970	{
2971	NOREF(pVCpu); NOREF(pCtx);
2972
2973	pVmcb->ctrl.EventInject.u = pEvent->u;
2974	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
2975
2976	Log4(("hmR0SvmInjectEventVmcb: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
2977	pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
2978	}
2979
2980
2981
2982	/**
2983	* Converts any TRPM trap into a pending HM event. This is typically used when
2984	* entering from ring-3 (not longjmp returns).
2985	*
2986	* @param pVCpu The cross context virtual CPU structure.
2987	*/
2988	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
2989	{
2990	Assert(TRPMHasTrap(pVCpu));
2991	Assert(!pVCpu->hm.s.Event.fPending);
2992
2993	uint8_t uVector;
2994	TRPMEVENT enmTrpmEvent;
2995	RTGCUINT uErrCode;
2996	RTGCUINTPTR GCPtrFaultAddress;
2997	uint8_t cbInstr;
2998
2999	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
3000	AssertRC(rc);
3001
3002	SVMEVENT Event;
3003	Event.u = 0;
3004	Event.n.u1Valid = 1;
3005	Event.n.u8Vector = uVector;
3006
3007	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3008	if (enmTrpmEvent == TRPM_TRAP)
3009	{
3010	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3011	switch (uVector)
3012	{
3013	case X86_XCPT_NMI:
3014	{
3015	Event.n.u3Type = SVM_EVENT_NMI;
3016	break;
3017	}
3018
3019	case X86_XCPT_PF:
3020	case X86_XCPT_DF:
3021	case X86_XCPT_TS:
3022	case X86_XCPT_NP:
3023	case X86_XCPT_SS:
3024	case X86_XCPT_GP:
3025	case X86_XCPT_AC:
3026	{
3027	Event.n.u1ErrorCodeValid = 1;
3028	Event.n.u32ErrorCode = uErrCode;
3029	break;
3030	}
3031	}
3032	}
3033	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3034	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3035	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3036	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3037	else
3038	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3039
3040	rc = TRPMResetTrap(pVCpu);
3041	AssertRC(rc);
3042
3043	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3044	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3045
3046	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3047	}
3048
3049
3050	/**
3051	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3052	* AMD-V to execute any instruction.
3053	*
3054	* @param pVCpu The cross context virtual CPU structure.
3055	*/
3056	static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
3057	{
3058	Assert(pVCpu->hm.s.Event.fPending);
3059	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3060
3061	SVMEVENT Event;
3062	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3063
3064	uint8_t uVector = Event.n.u8Vector;
3065	uint8_t uVectorType = Event.n.u3Type;
3066	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event);
3067
3068	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, uVectorType));
3069
3070	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3071	AssertRC(rc);
3072
3073	if (Event.n.u1ErrorCodeValid)
3074	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3075
3076	if ( uVectorType == SVM_EVENT_EXCEPTION
3077	&& uVector == X86_XCPT_PF)
3078	{
3079	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3080	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3081	}
3082	else if (uVectorType == SVM_EVENT_SOFTWARE_INT)
3083	{
3084	AssertMsg( uVectorType == SVM_EVENT_SOFTWARE_INT
3085	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
3086	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
3087	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3088	}
3089	pVCpu->hm.s.Event.fPending = false;
3090	}
3091
3092
3093	/**
3094	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3095	* now.
3096	*
3097	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3098	* @param pVCpu The cross context virtual CPU structure.
3099	* @param pCtx Pointer to the guest-CPU context.
3100	*
3101	* @remarks No-long-jump zone!!!
3102	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3103	*/
3104	DECLINLINE(bool) hmR0SvmIsIntrShadowActive(PVMCPU pVCpu, PCPUMCTX pCtx)
3105	{
3106	/*
3107	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3108	* inhibit interrupts or clear any existing interrupt-inhibition.
3109	*/
3110	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3111	{
3112	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3113	{
3114	/*
3115	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3116	* AMD-V, the flag's condition to be cleared is met and thus the cleared state is correct.
3117	*/
3118	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3119	return false;
3120	}
3121	return true;
3122	}
3123	return false;
3124	}
3125
3126
3127	/**
3128	* Sets the virtual interrupt intercept control in the VMCB.
3129	*
3130	* @param pVmcb Pointer to the VM control block.
3131	*/
3132	DECLINLINE(void) hmR0SvmSetVirtIntrIntercept(PSVMVMCB pVmcb)
3133	{
3134	/*
3135	* When AVIC isn't supported, indicate that a virtual interrupt is pending and to
3136	* cause a #VMEXIT when the guest is ready to accept interrupts. At #VMEXIT, we
3137	* then get the interrupt from the APIC (updating ISR at the right time) and
3138	* inject the interrupt.
3139	*
3140	* With AVIC is supported, we could make use of the asynchronously delivery without
3141	* #VMEXIT and we would be passing the AVIC page to SVM.
3142	*/
3143	if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3144	{
3145	Assert(pVmcb->ctrl.IntCtrl.n.u1VIrqPending == 0);
3146	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3147	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VINTR;
3148	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_TPR);
3149	Log4(("Set VINTR intercept\n"));
3150	}
3151	}
3152
3153
3154	/**
3155	* Clears the virtual interrupt intercept control in the VMCB as
3156	* we are figured the guest is unable process any interrupts
3157	* at this point of time.
3158	*
3159	* @param pVmcb Pointer to the VM control block.
3160	*/
3161	DECLINLINE(void) hmR0SvmClearVirtIntrIntercept(PSVMVMCB pVmcb)
3162	{
3163	if (pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR)
3164	{
3165	Assert(pVmcb->ctrl.IntCtrl.n.u1VIrqPending == 1);
3166	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 0;
3167	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_VINTR;
3168	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_TPR);
3169	Log4(("Cleared VINTR intercept\n"));
3170	}
3171	}
3172
3173
3174	/**
3175	* Sets the IRET intercept control in the VMCB which instructs AMD-V to cause a
3176	* \#VMEXIT as soon as a guest starts executing an IRET. This is used to unblock
3177	* virtual NMIs.
3178	*
3179	* @param pVmcb Pointer to the VM control block.
3180	*/
3181	DECLINLINE(void) hmR0SvmSetIretIntercept(PSVMVMCB pVmcb)
3182	{
3183	if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_IRET))
3184	{
3185	pVmcb->ctrl.u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_IRET;
3186	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3187
3188	Log4(("Setting IRET intercept\n"));
3189	}
3190	}
3191
3192
3193	/**
3194	* Clears the IRET intercept control in the VMCB.
3195	*
3196	* @param pVmcb Pointer to the VM control block.
3197	*/
3198	DECLINLINE(void) hmR0SvmClearIretIntercept(PSVMVMCB pVmcb)
3199	{
3200	if (pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_IRET)
3201	{
3202	pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_IRET;
3203	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS);
3204
3205	Log4(("Clearing IRET intercept\n"));
3206	}
3207	}
3208
3209	#ifdef VBOX_WITH_NESTED_HWVIRT
3210
3211
3212	/**
3213	* Evaluates the event to be delivered to the nested-guest and sets it as the
3214	* pending event.
3215	*
3216	* @returns VBox strict status code.
3217	* @param pVCpu The cross context virtual CPU structure.
3218	* @param pCtx Pointer to the guest-CPU context.
3219	*/
3220	static VBOXSTRICTRC hmR0SvmEvaluatePendingEventNested(PVMCPU pVCpu, PCPUMCTX pCtx)
3221	{
3222	Log4Func(("\n"));
3223
3224	Assert(!pVCpu->hm.s.Event.fPending);
3225
3226	bool const fGif = pCtx->hwvirt.svm.fGif;
3227	if (fGif)
3228	{
3229	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
3230
3231	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3232
3233	/*
3234	* Check if the nested-guest can receive NMIs.
3235	* NMIs are higher priority than regular interrupts.
3236	*/
3237	/** @todo SMI. SMIs take priority over NMIs. */
3238	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI))
3239	{
3240	bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3241	if (fBlockNmi)
3242	hmR0SvmSetIretIntercept(pVmcbNstGst);
3243	else if (fIntShadow)
3244	{
3245	/** @todo Figure this out, how we shall manage virt. intercept if the
3246	* nested-guest already has one set and/or if we really need it? */
3247	//hmR0SvmSetVirtIntrIntercept(pVmcbNstGst);
3248	}
3249	else
3250	{
3251	Log4(("Pending NMI\n"));
3252
3253	SVMEVENT Event;
3254	Event.u = 0;
3255	Event.n.u1Valid = 1;
3256	Event.n.u8Vector = X86_XCPT_NMI;
3257	Event.n.u3Type = SVM_EVENT_NMI;
3258
3259	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3260	hmR0SvmSetIretIntercept(pVmcbNstGst);
3261	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3262	return VINF_SUCCESS;
3263	}
3264	}
3265
3266	/*
3267	* Check if the nested-guest can receive external interrupts (generated by
3268	* the guest's PIC/APIC).
3269	*
3270	* External intercepts, NMI, SMI etc. from the physical CPU are -always- intercepted
3271	* when executing using hardware-assisted SVM, see HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS.
3272	*
3273	* External interrupts that are generated for the outer guest may be intercepted
3274	* depending on how the nested-guest VMCB was programmed by guest software.
3275	*
3276	* Physical interrupts always take priority over virtual interrupts,
3277	* see AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3278	*/
3279	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
3280	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3281	&& !fIntShadow
3282	&& !pVCpu->hm.s.fSingleInstruction
3283	&& CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx))
3284	{
3285	if (pVmcbNstGstCache->u64InterceptCtrl & SVM_CTRL_INTERCEPT_INTR)
3286	{
3287	Log4(("Intercepting external interrupt -> #VMEXIT\n"));
3288	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3289	}
3290
3291	uint8_t u8Interrupt;
3292	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3293	if (RT_SUCCESS(rc))
3294	{
3295	Log4(("Injecting external interrupt u8Interrupt=%#x\n", u8Interrupt));
3296
3297	SVMEVENT Event;
3298	Event.u = 0;
3299	Event.n.u1Valid = 1;
3300	Event.n.u8Vector = u8Interrupt;
3301	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3302
3303	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3304	}
3305	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3306	{
3307	/*
3308	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3309	* updated eventually when the TPR is written by the guest.
3310	*/
3311	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3312	}
3313	else
3314	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3315	}
3316
3317	/*
3318	* Check if the nested-guest is intercepting virtual (using V_IRQ and related fields)
3319	* interrupt injection. The virtual interrupt injection itself, if any, will be done
3320	* by the physical CPU.
3321	*/
3322	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)
3323	&& (pVmcbNstGstCache->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR)
3324	&& CPUMCanSvmNstGstTakeVirtIntr(pCtx))
3325	{
3326	Log4(("Intercepting virtual interrupt -> #VMEXIT\n"));
3327	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_VINTR, 0, 0);
3328	}
3329	}
3330
3331	return VINF_SUCCESS;
3332	}
3333	#endif
3334
3335
3336	/**
3337	* Evaluates the event to be delivered to the guest and sets it as the pending
3338	* event.
3339	*
3340	* @param pVCpu The cross context virtual CPU structure.
3341	* @param pCtx Pointer to the guest-CPU context.
3342	*
3343	* @remarks Don't use this function when we are actively executing a
3344	* nested-guest, use hmR0SvmEvaluatePendingEventNested instead.
3345	*/
3346	static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
3347	{
3348	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3349	Assert(!pVCpu->hm.s.Event.fPending);
3350
3351	#ifdef VBOX_WITH_NESTED_HWVIRT
3352	bool const fGif = pCtx->hwvirt.svm.fGif;
3353	#else
3354	bool const fGif = true;
3355	#endif
3356	Log4Func(("fGif=%RTbool\n", fGif));
3357
3358	/*
3359	* If the global interrupt flag (GIF) isn't set, even NMIs and other events are blocked.
3360	* See AMD spec. Table 15-10. "Effect of the GIF on Interrupt Handling".
3361	*/
3362	if (fGif)
3363	{
3364	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3365	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3366	bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3367	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3368
3369	Log4Func(("fGif=%RTbool fBlockInt=%RTbool fIntShadow=%RTbool APIC/PIC_Pending=%RTbool\n", fGif, fBlockInt, fIntShadow,
3370	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
3371
3372	/** @todo SMI. SMIs take priority over NMIs. */
3373	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
3374	{
3375	if (fBlockNmi)
3376	hmR0SvmSetIretIntercept(pVmcb);
3377	else if (fIntShadow)
3378	hmR0SvmSetVirtIntrIntercept(pVmcb);
3379	else
3380	{
3381	Log4(("Pending NMI\n"));
3382
3383	SVMEVENT Event;
3384	Event.u = 0;
3385	Event.n.u1Valid = 1;
3386	Event.n.u8Vector = X86_XCPT_NMI;
3387	Event.n.u3Type = SVM_EVENT_NMI;
3388
3389	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3390	hmR0SvmSetIretIntercept(pVmcb);
3391	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3392	return;
3393	}
3394	}
3395	else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3396	&& !pVCpu->hm.s.fSingleInstruction)
3397	{
3398	/*
3399	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
3400	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
3401	*/
3402	if ( !fBlockInt
3403	&& !fIntShadow)
3404	{
3405	uint8_t u8Interrupt;
3406	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3407	if (RT_SUCCESS(rc))
3408	{
3409	Log4(("Injecting external interrupt u8Interrupt=%#x\n", u8Interrupt));
3410
3411	SVMEVENT Event;
3412	Event.u = 0;
3413	Event.n.u1Valid = 1;
3414	Event.n.u8Vector = u8Interrupt;
3415	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3416
3417	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3418	}
3419	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3420	{
3421	/*
3422	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3423	* updated eventually when the TPR is written by the guest.
3424	*/
3425	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3426	}
3427	else
3428	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3429	}
3430	else
3431	hmR0SvmSetVirtIntrIntercept(pVmcb);
3432	}
3433	}
3434	}
3435
3436
3437	/**
3438	* Injects any pending events into the guest or nested-guest.
3439	*
3440	* @param pVCpu The cross context virtual CPU structure.
3441	* @param pCtx Pointer to the guest-CPU context.
3442	* @param pVmcb Pointer to the VM control block.
3443	*/
3444	static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
3445	{
3446	Assert(!TRPMHasTrap(pVCpu));
3447	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3448
3449	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3450
3451	/*
3452	* When executing the nested-guest, we avoid assertions on whether the
3453	* event injection is valid purely based on EFLAGS, as V_INTR_MASKING
3454	* affects the interpretation of interruptibility (see CPUMCanSvmNstGstTakePhysIntr).
3455	*/
3456	#ifndef VBOX_WITH_NESTED_HWVIRT
3457	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3458	#endif
3459
3460	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
3461	{
3462	SVMEVENT Event;
3463	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3464	Assert(Event.n.u1Valid);
3465
3466	#ifndef VBOX_WITH_NESTED_HWVIRT
3467	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3468	{
3469	Assert(!fBlockInt);
3470	Assert(!fIntShadow);
3471	}
3472	else if (Event.n.u3Type == SVM_EVENT_NMI)
3473	Assert(!fIntShadow);
3474	NOREF(fBlockInt);
3475	#else
3476	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3477	#endif
3478
3479	Log4(("Injecting pending HM event\n"));
3480	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, pCtx, &Event);
3481	pVCpu->hm.s.Event.fPending = false;
3482
3483	#ifdef VBOX_WITH_STATISTICS
3484	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3485	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3486	else
3487	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3488	#endif
3489	}
3490
3491	/*
3492	* Update the guest interrupt shadow in the guest or nested-guest VMCB.
3493	*
3494	* For nested-guests: We need to update it too for the scenario where IEM executes
3495	* the nested-guest but execution later continues here with an interrupt shadow active.
3496	*/
3497	pVmcb->ctrl.u64IntShadow = !!fIntShadow;
3498	}
3499
3500
3501	/**
3502	* Reports world-switch error and dumps some useful debug info.
3503	*
3504	* @param pVM The cross context VM structure.
3505	* @param pVCpu The cross context virtual CPU structure.
3506	* @param rcVMRun The return code from VMRUN (or
3507	* VERR_SVM_INVALID_GUEST_STATE for invalid
3508	* guest-state).
3509	* @param pCtx Pointer to the guest-CPU context.
3510	*/
3511	static void hmR0SvmReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx)
3512	{
3513	NOREF(pCtx);
3514	HMSVM_ASSERT_PREEMPT_SAFE();
3515	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3516	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3517
3518	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3519	{
3520	hmR0DumpRegs(pVM, pVCpu, pCtx); NOREF(pVM);
3521	#ifdef VBOX_STRICT
3522	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3523	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3524	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3525	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3526	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3527	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3528	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3529	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3530	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3531	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3532
3533	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3534	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3535	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3536
3537	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3538	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3539	Log4(("ctrl.IntCtrl.u7Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u7Reserved));
3540	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3541	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3542	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3543	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3544	Log4(("ctrl.IntCtrl.u6Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3545	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3546	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3547
3548	Log4(("ctrl.u64IntShadow %#RX64\n", pVmcb->ctrl.u64IntShadow));
3549	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3550	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3551	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3552	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3553	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3554	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3555	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3556	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3557	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3558	Log4(("ctrl.u1NestedPaging %RTbool\n", pVmcb->ctrl.u1NestedPaging));
3559	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3560	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3561	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3562	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3563	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3564	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3565
3566	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3567	Log4(("ctrl.u1Lbrvirt %RTbool\n", pVmcb->ctrl.u1LbrVirt));
3568
3569	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3570	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3571	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3572	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3573	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3574	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3575	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3576	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3577	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3578	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3579	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3580	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3581	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3582	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3583	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3584	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3585	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3586	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3587	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3588	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3589
3590	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3591	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3592
3593	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3594	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3595	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3596	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3597
3598	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3599	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3600
3601	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3602	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3603	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3604	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3605
3606	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3607	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3608	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3609	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3610	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3611	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3612	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3613
3614	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3615	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3616	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3617	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3618
3619	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3620	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3621	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3622
3623	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3624	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3625	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3626	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3627	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3628	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3629	Log4(("guest.u64GPAT %#RX64\n", pVmcb->guest.u64GPAT));
3630	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3631	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3632	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3633	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3634	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3635	#endif /* VBOX_STRICT */
3636	}
3637	else
3638	Log4(("hmR0SvmReportWorldSwitchError: rcVMRun=%d\n", rcVMRun));
3639
3640	NOREF(pVmcb);
3641	}
3642
3643
3644	/**
3645	* Check per-VM and per-VCPU force flag actions that require us to go back to
3646	* ring-3 for one reason or another.
3647	*
3648	* @returns VBox status code (information status code included).
3649	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3650	* ring-3.
3651	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3652	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3653	* interrupts)
3654	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3655	* all EMTs to be in ring-3.
3656	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3657	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3658	* to the EM loop.
3659	*
3660	* @param pVM The cross context VM structure.
3661	* @param pVCpu The cross context virtual CPU structure.
3662	* @param pCtx Pointer to the guest-CPU context.
3663	*/
3664	static int hmR0SvmCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3665	{
3666	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3667
3668	/* On AMD-V we don't need to update CR3, PAE PDPES lazily. See hmR0SvmSaveGuestState(). */
3669	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3670	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
3671
3672	/* Update pending interrupts into the APIC's IRR. */
3673	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3674	APICUpdatePendingInterrupts(pVCpu);
3675
3676	if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
3677	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3678	\|\| VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
3679	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3680	{
3681	/* Pending PGM C3 sync. */
3682	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3683	{
3684	int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3685	if (rc != VINF_SUCCESS)
3686	{
3687	Log4(("hmR0SvmCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3688	return rc;
3689	}
3690	}
3691
3692	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3693	/* -XXX- what was that about single stepping? */
3694	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
3695	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3696	{
3697	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3698	int rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3699	Log4(("hmR0SvmCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3700	return rc;
3701	}
3702
3703	/* Pending VM request packets, such as hardware interrupts. */
3704	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
3705	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
3706	{
3707	Log4(("hmR0SvmCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
3708	return VINF_EM_PENDING_REQUEST;
3709	}
3710
3711	/* Pending PGM pool flushes. */
3712	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3713	{
3714	Log4(("hmR0SvmCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
3715	return VINF_PGM_POOL_FLUSH_PENDING;
3716	}
3717
3718	/* Pending DMA requests. */
3719	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
3720	{
3721	Log4(("hmR0SvmCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
3722	return VINF_EM_RAW_TO_R3;
3723	}
3724	}
3725
3726	return VINF_SUCCESS;
3727	}
3728
3729
3730	#ifdef VBOX_WITH_NESTED_HWVIRT
3731	/**
3732	* Does the preparations before executing nested-guest code in AMD-V.
3733	*
3734	* @returns VBox status code (informational status codes included).
3735	* @retval VINF_SUCCESS if we can proceed with running the guest.
3736	* @retval VINF_* scheduling changes, we have to go back to ring-3.
3737	*
3738	* @param pVM The cross context VM structure.
3739	* @param pVCpu The cross context virtual CPU structure.
3740	* @param pCtx Pointer to the guest-CPU context.
3741	* @param pSvmTransient Pointer to the SVM transient structure.
3742	*
3743	* @remarks Same caveats regarding longjumps as hmR0SvmPreRunGuest applies.
3744	* @sa hmR0SvmPreRunGuest.
3745	*/
3746	static int hmR0SvmPreRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3747	{
3748	HMSVM_ASSERT_PREEMPT_SAFE();
3749
3750	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3751	{
3752	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
3753	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
3754	return VINF_EM_RESCHEDULE_REM;
3755	#endif
3756	}
3757	else
3758	return VINF_SVM_VMEXIT;
3759
3760	/* Check force flag actions that might require us to go back to ring-3. */
3761	int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
3762	if (rc != VINF_SUCCESS)
3763	return rc;
3764
3765	if (TRPMHasTrap(pVCpu))
3766	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
3767	else if (!pVCpu->hm.s.Event.fPending)
3768	{
3769	VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEventNested(pVCpu, pCtx);
3770	if (rcStrict != VINF_SUCCESS)
3771	return VBOXSTRICTRC_VAL(rcStrict);
3772	}
3773
3774	/*
3775	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
3776	* Just do it in software, see @bugref{8411}.
3777	* NB: If we could continue a task switch exit we wouldn't need to do this.
3778	*/
3779	if (RT_UNLIKELY( !pVM->hm.s.svm.u32Features
3780	&& pVCpu->hm.s.Event.fPending
3781	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
3782	{
3783	return VINF_EM_RAW_INJECT_TRPM_EVENT;
3784	}
3785
3786	/*
3787	* Load the nested-guest state.
3788	*/
3789	rc = hmR0SvmLoadGuestStateNested(pVCpu, pCtx);
3790	AssertRCReturn(rc, rc);
3791	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull); /** @todo Get new STAM counter for this? */
3792
3793	/* Ensure we've cached (and hopefully modified) the VMCB for execution using hardware SVM. */
3794	Assert(pCtx->hwvirt.svm.fHMCachedVmcb);
3795
3796	/*
3797	* No longjmps to ring-3 from this point on!!!
3798	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
3799	* This also disables flushing of the R0-logger instance (if any).
3800	*/
3801	VMMRZCallRing3Disable(pVCpu);
3802
3803	/*
3804	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
3805	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
3806	*
3807	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
3808	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
3809	*
3810	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
3811	* executing guest code.
3812	*/
3813	pSvmTransient->fEFlags = ASMIntDisableFlags();
3814	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
3815	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3816	{
3817	ASMSetFlags(pSvmTransient->fEFlags);
3818	VMMRZCallRing3Enable(pVCpu);
3819	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3820	return VINF_EM_RAW_TO_R3;
3821	}
3822	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3823	{
3824	ASMSetFlags(pSvmTransient->fEFlags);
3825	VMMRZCallRing3Enable(pVCpu);
3826	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3827	return VINF_EM_RAW_INTERRUPT;
3828	}
3829
3830	/*
3831	* If we are injecting an NMI, we must set VMCPU_FF_BLOCK_NMIS only when we are going to execute
3832	* guest code for certain (no exits to ring-3). Otherwise, we could re-read the flag on re-entry into
3833	* AMD-V and conclude that NMI inhibition is active when we have not even delivered the NMI.
3834	*
3835	* With VT-x, this is handled by the Guest interruptibility information VMCS field which will set the
3836	* VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
3837	*/
3838	if (pVCpu->hm.s.Event.fPending)
3839	{
3840	SVMEVENT Event;
3841	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3842	if ( Event.n.u1Valid
3843	&& Event.n.u3Type == SVM_EVENT_NMI
3844	&& Event.n.u8Vector == X86_XCPT_NMI
3845	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3846	{
3847	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3848	}
3849	}
3850
3851	return VINF_SUCCESS;
3852	}
3853	#endif
3854
3855
3856	/**
3857	* Does the preparations before executing guest code in AMD-V.
3858	*
3859	* This may cause longjmps to ring-3 and may even result in rescheduling to the
3860	* recompiler. We must be cautious what we do here regarding committing
3861	* guest-state information into the VMCB assuming we assuredly execute the guest
3862	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
3863	* clearing the common-state (TRPM/forceflags), we must undo those changes so
3864	* that the recompiler can (and should) use them when it resumes guest
3865	* execution. Otherwise such operations must be done when we can no longer
3866	* exit to ring-3.
3867	*
3868	* @returns VBox status code (informational status codes included).
3869	* @retval VINF_SUCCESS if we can proceed with running the guest.
3870	* @retval VINF_* scheduling changes, we have to go back to ring-3.
3871	*
3872	* @param pVM The cross context VM structure.
3873	* @param pVCpu The cross context virtual CPU structure.
3874	* @param pCtx Pointer to the guest-CPU context.
3875	* @param pSvmTransient Pointer to the SVM transient structure.
3876	*/
3877	static int hmR0SvmPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3878	{
3879	HMSVM_ASSERT_PREEMPT_SAFE();
3880	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3881
3882	/* Check force flag actions that might require us to go back to ring-3. */
3883	int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
3884	if (rc != VINF_SUCCESS)
3885	return rc;
3886
3887	if (TRPMHasTrap(pVCpu))
3888	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
3889	else if (!pVCpu->hm.s.Event.fPending)
3890	hmR0SvmEvaluatePendingEvent(pVCpu, pCtx);
3891
3892	/*
3893	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
3894	* Just do it in software, see @bugref{8411}.
3895	* NB: If we could continue a task switch exit we wouldn't need to do this.
3896	*/
3897	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending && (((pVCpu->hm.s.Event.u64IntInfo >> 8) & 7) == SVM_EVENT_NMI)))
3898	if (RT_UNLIKELY(!pVM->hm.s.svm.u32Features))
3899	return VINF_EM_RAW_INJECT_TRPM_EVENT;
3900
3901	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
3902	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
3903	#endif
3904
3905	/* Load the guest bits that are not shared with the host in any way since we can longjmp or get preempted. */
3906	rc = hmR0SvmLoadGuestState(pVM, pVCpu, pCtx);
3907	AssertRCReturn(rc, rc);
3908	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3909
3910	/*
3911	* If we're not intercepting TPR changes in the guest, save the guest TPR before the world-switch
3912	* so we can update it on the way back if the guest changed the TPR.
3913	*/
3914	if (pVCpu->hm.s.svm.fSyncVTpr)
3915	{
3916	if (pVM->hm.s.fTPRPatchingActive)
3917	pSvmTransient->u8GuestTpr = pCtx->msrLSTAR;
3918	else
3919	{
3920	PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3921	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
3922	}
3923	}
3924
3925	/*
3926	* No longjmps to ring-3 from this point on!!!
3927	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
3928	* This also disables flushing of the R0-logger instance (if any).
3929	*/
3930	VMMRZCallRing3Disable(pVCpu);
3931
3932	/*
3933	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
3934	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
3935	*
3936	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
3937	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
3938	*
3939	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
3940	* executing guest code.
3941	*/
3942	pSvmTransient->fEFlags = ASMIntDisableFlags();
3943	if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
3944	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3945	{
3946	ASMSetFlags(pSvmTransient->fEFlags);
3947	VMMRZCallRing3Enable(pVCpu);
3948	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3949	return VINF_EM_RAW_TO_R3;
3950	}
3951	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3952	{
3953	ASMSetFlags(pSvmTransient->fEFlags);
3954	VMMRZCallRing3Enable(pVCpu);
3955	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3956	return VINF_EM_RAW_INTERRUPT;
3957	}
3958
3959	/*
3960	* If we are injecting an NMI, we must set VMCPU_FF_BLOCK_NMIS only when we are going to execute
3961	* guest code for certain (no exits to ring-3). Otherwise, we could re-read the flag on re-entry into
3962	* AMD-V and conclude that NMI inhibition is active when we have not even delivered the NMI.
3963	*
3964	* With VT-x, this is handled by the Guest interruptibility information VMCS field which will set the
3965	* VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
3966	*/
3967	if (pVCpu->hm.s.Event.fPending)
3968	{
3969	SVMEVENT Event;
3970	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3971	if ( Event.n.u1Valid
3972	&& Event.n.u3Type == SVM_EVENT_NMI
3973	&& Event.n.u8Vector == X86_XCPT_NMI
3974	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3975	{
3976	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3977	}
3978	}
3979
3980	return VINF_SUCCESS;
3981	}
3982
3983
3984	#ifdef VBOX_WITH_NESTED_HWVIRT
3985	/**
3986	* Prepares to run nested-guest code in AMD-V and we've committed to doing so. This
3987	* means there is no backing out to ring-3 or anywhere else at this point.
3988	*
3989	* @param pVM The cross context VM structure.
3990	* @param pVCpu The cross context virtual CPU structure.
3991	* @param pCtx Pointer to the guest-CPU context.
3992	* @param pSvmTransient Pointer to the SVM transient structure.
3993	*
3994	* @remarks Called with preemption disabled.
3995	* @remarks No-long-jump zone!!!
3996	*/
3997	static void hmR0SvmPreRunGuestCommittedNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3998	{
3999	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4000	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4002	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4003
4004	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4005	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4006
4007	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4008	hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcbNstGst);
4009
4010	if ( pVCpu->hm.s.fPreloadGuestFpu
4011	&& !CPUMIsGuestFPUStateActive(pVCpu))
4012	{
4013	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4014	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4015	}
4016
4017	/* Load the state shared between host and nested-guest (FPU, debug). */
4018	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4019	hmR0SvmLoadSharedState(pVCpu, pVmcbNstGst, pCtx);
4020
4021	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4022	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4023
4024	/* Setup TSC offsetting. */
4025	RTCPUID idCurrentCpu = hmR0GetCurrentCpu()->idCpu;
4026	if ( pSvmTransient->fUpdateTscOffsetting
4027	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
4028	{
4029	hmR0SvmUpdateTscOffsettingNested(pVM, pVCpu, pCtx, pVmcbNstGst);
4030	pSvmTransient->fUpdateTscOffsetting = false;
4031	}
4032
4033	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4034	if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
4035	pVmcbNstGst->ctrl.u32VmcbCleanBits = 0;
4036
4037	/* Store status of the shared guest-host state at the time of VMRUN. */
4038	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4039	if (CPUMIsGuestInLongModeEx(pCtx))
4040	{
4041	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4042	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4043	}
4044	else
4045	#endif
4046	{
4047	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4048	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4049	}
4050	pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
4051
4052	/* The TLB flushing would've already been setup by the nested-hypervisor. */
4053	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4054	hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcbNstGst);
4055	Assert(hmR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
4056
4057	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4058
4059	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4060	to start executing. */
4061
4062	/*
4063	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4064	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4065	*
4066	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4067	*/
4068	uint8_t pbMsrBitmap = (uint8_t )pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4069	if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4070	&& !(pVmcbNstGst->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4071	{
4072	hmR0SvmSetMsrPermission(pVmcbNstGst, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4073	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4074	uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4075	if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
4076	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTscAux);
4077	pSvmTransient->fRestoreTscAuxMsr = true;
4078	}
4079	else
4080	{
4081	hmR0SvmSetMsrPermission(pVmcbNstGst, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4082	pSvmTransient->fRestoreTscAuxMsr = false;
4083	}
4084
4085	/*
4086	* If VMCB Clean bits isn't supported by the CPU or exposed by the guest,
4087	* mark all state-bits as dirty indicating to the CPU to re-load from VMCB.
4088	*/
4089	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4090	if (!fSupportsVmcbCleanBits)
4091	pVmcbNstGst->ctrl.u32VmcbCleanBits = 0;
4092	}
4093	#endif
4094
4095
4096	/**
4097	* Prepares to run guest code in AMD-V and we've committed to doing so. This
4098	* means there is no backing out to ring-3 or anywhere else at this
4099	* point.
4100	*
4101	* @param pVM The cross context VM structure.
4102	* @param pVCpu The cross context virtual CPU structure.
4103	* @param pCtx Pointer to the guest-CPU context.
4104	* @param pSvmTransient Pointer to the SVM transient structure.
4105	*
4106	* @remarks Called with preemption disabled.
4107	* @remarks No-long-jump zone!!!
4108	*/
4109	static void hmR0SvmPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4110	{
4111	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4112	Assert(VMMR0IsLogFlushDisabled(pVCpu));
4113	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4114	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
4115
4116	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4117	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4118
4119	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4120	hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcb);
4121
4122	if ( pVCpu->hm.s.fPreloadGuestFpu
4123	&& !CPUMIsGuestFPUStateActive(pVCpu))
4124	{
4125	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4126	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4127	}
4128
4129	/* Load the state shared between host and guest (FPU, debug). */
4130	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4131	hmR0SvmLoadSharedState(pVCpu, pVmcb, pCtx);
4132
4133	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4134	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4135
4136	/* Setup TSC offsetting. */
4137	RTCPUID idCurrentCpu = hmR0GetCurrentCpu()->idCpu;
4138	if ( pSvmTransient->fUpdateTscOffsetting
4139	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
4140	{
4141	hmR0SvmUpdateTscOffsetting(pVM, pVCpu, pVmcb);
4142	pSvmTransient->fUpdateTscOffsetting = false;
4143	}
4144
4145	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4146	if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
4147	pVmcb->ctrl.u32VmcbCleanBits = 0;
4148
4149	/* Store status of the shared guest-host state at the time of VMRUN. */
4150	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4151	if (CPUMIsGuestInLongModeEx(pCtx))
4152	{
4153	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4154	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4155	}
4156	else
4157	#endif
4158	{
4159	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4160	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4161	}
4162	pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
4163
4164	/* Flush the appropriate tagged-TLB entries. */
4165	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4166	hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcb);
4167	Assert(hmR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
4168
4169	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4170
4171	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4172	to start executing. */
4173
4174	/*
4175	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4176	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4177	*
4178	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4179	*/
4180	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4181	if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4182	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4183	{
4184	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4185	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4186	uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4187	if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
4188	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTscAux);
4189	pSvmTransient->fRestoreTscAuxMsr = true;
4190	}
4191	else
4192	{
4193	hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4194	pSvmTransient->fRestoreTscAuxMsr = false;
4195	}
4196
4197	/* If VMCB Clean bits isn't supported by the CPU, simply mark all state-bits as dirty, indicating (re)load-from-VMCB. */
4198	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4199	if (!fSupportsVmcbCleanBits)
4200	pVmcb->ctrl.u32VmcbCleanBits = 0;
4201	}
4202
4203
4204	/**
4205	* Wrapper for running the guest code in AMD-V.
4206	*
4207	* @returns VBox strict status code.
4208	* @param pVM The cross context VM structure.
4209	* @param pVCpu The cross context virtual CPU structure.
4210	* @param pCtx Pointer to the guest-CPU context.
4211	*
4212	* @remarks No-long-jump zone!!!
4213	*/
4214	DECLINLINE(int) hmR0SvmRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4215	{
4216	/*
4217	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4218	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4219	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4220	*/
4221	#ifdef VBOX_WITH_KERNEL_USING_XMM
4222	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4223	pVCpu->hm.s.svm.pfnVMRun);
4224	#else
4225	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4226	#endif
4227	}
4228
4229
4230	#ifdef VBOX_WITH_NESTED_HWVIRT
4231	/**
4232	* Wrapper for running the nested-guest code in AMD-V.
4233	*
4234	* @returns VBox strict status code.
4235	* @param pVM The cross context VM structure.
4236	* @param pVCpu The cross context virtual CPU structure.
4237	* @param pCtx Pointer to the guest-CPU context.
4238	*
4239	* @remarks No-long-jump zone!!!
4240	*/
4241	DECLINLINE(int) hmR0SvmRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4242	{
4243	/*
4244	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4245	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4246	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4247	*/
4248	#ifdef VBOX_WITH_KERNEL_USING_XMM
4249	return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4250	pVCpu->hm.s.svm.pfnVMRun);
4251	#else
4252	return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4253	#endif
4254	}
4255
4256
4257	/**
4258	* Performs some essential restoration of state after running nested-guest code in
4259	* AMD-V.
4260	*
4261	* @param pVM The cross context VM structure.
4262	* @param pVCpu The cross context virtual CPU structure.
4263	* @param pMixedCtx Pointer to the nested-guest-CPU context. The data maybe
4264	* out-of-sync. Make sure to update the required fields
4265	* before using them.
4266	* @param pSvmTransient Pointer to the SVM transient structure.
4267	* @param rcVMRun Return code of VMRUN.
4268	*
4269	* @remarks Called with interrupts disabled.
4270	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4271	* unconditionally when it is safe to do so.
4272	*/
4273	static void hmR0SvmPostRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4274	{
4275	RT_NOREF(pVM);
4276	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4277
4278	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4279	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4280
4281	/* TSC read must be done early for maximum accuracy. */
4282	PSVMVMCB pVmcbNstGst = pMixedCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4283	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4284	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
4285	if (!(pVmcbNstGstCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4286	{
4287	/*
4288	* Undo what we did in hmR0SvmUpdateTscOffsettingNested() but don't restore the
4289	* nested-guest VMCB TSC offset here. It shall eventually be restored on #VMEXIT
4290	* later by HMSvmNstGstVmExitNotify().
4291	*/
4292	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcbNstGstCtrl->u64TSCOffset - pVmcbNstGstCache->u64TSCOffset);
4293	}
4294
4295	if (pSvmTransient->fRestoreTscAuxMsr)
4296	{
4297	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4298	CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4299	if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4300	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4301	}
4302
4303	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4304	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4305	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4306
4307	Assert(!(ASMGetFlags() & X86_EFL_IF));
4308	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4309	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4310
4311	/* Mark the VMCB-state cache as unmodified by VMM. */
4312	pVmcbNstGstCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL;
4313
4314	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4315	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4316	{
4317	Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4318	return;
4319	}
4320
4321	pSvmTransient->u64ExitCode = pVmcbNstGstCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4322	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcbNstGstCtrl->u64ExitCode);/* Update the #VMEXIT history array. */
4323	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4324	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4325
4326	Assert(!pVCpu->hm.s.svm.fSyncVTpr);
4327	hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcbNstGst); /* Save the nested-guest state from the VMCB to the
4328	guest-CPU context. */
4329	}
4330	#endif
4331
4332	/**
4333	* Performs some essential restoration of state after running guest code in
4334	* AMD-V.
4335	*
4336	* @param pVM The cross context VM structure.
4337	* @param pVCpu The cross context virtual CPU structure.
4338	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4339	* out-of-sync. Make sure to update the required fields
4340	* before using them.
4341	* @param pSvmTransient Pointer to the SVM transient structure.
4342	* @param rcVMRun Return code of VMRUN.
4343	*
4344	* @remarks Called with interrupts disabled.
4345	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4346	* unconditionally when it is safe to do so.
4347	*/
4348	static void hmR0SvmPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4349	{
4350	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4351
4352	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4353	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4354
4355	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4356	pVmcb->ctrl.u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4357
4358	/* TSC read must be done early for maximum accuracy. */
4359	if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4360	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcb->ctrl.u64TSCOffset);
4361
4362	if (pSvmTransient->fRestoreTscAuxMsr)
4363	{
4364	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4365	CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4366	if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4367	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4368	}
4369
4370	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4371	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4372	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4373
4374	Assert(!(ASMGetFlags() & X86_EFL_IF));
4375	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4376	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4377
4378	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4379	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4380	{
4381	Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4382	return;
4383	}
4384
4385	pSvmTransient->u64ExitCode = pVmcb->ctrl.u64ExitCode; /* Save the #VMEXIT reason. */
4386	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcb->ctrl.u64ExitCode); /* Update the #VMEXIT history array. */
4387	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4388	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4389
4390	hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcb); /* Save the guest state from the VMCB to the guest-CPU context. */
4391
4392	if (RT_LIKELY(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID))
4393	{
4394	if (pVCpu->hm.s.svm.fSyncVTpr)
4395	{
4396	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4397	if ( pVM->hm.s.fTPRPatchingActive
4398	&& (pMixedCtx->msrLSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4399	{
4400	int rc = APICSetTpr(pVCpu, pMixedCtx->msrLSTAR & 0xff);
4401	AssertRC(rc);
4402	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4403	}
4404	else if (pSvmTransient->u8GuestTpr != pVmcb->ctrl.IntCtrl.n.u8VTPR)
4405	{
4406	int rc = APICSetTpr(pVCpu, pVmcb->ctrl.IntCtrl.n.u8VTPR << 4);
4407	AssertRC(rc);
4408	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4409	}
4410	}
4411	}
4412	}
4413
4414
4415	/**
4416	* Runs the guest code using AMD-V.
4417	*
4418	* @returns VBox status code.
4419	* @param pVM The cross context VM structure.
4420	* @param pVCpu The cross context virtual CPU structure.
4421	* @param pCtx Pointer to the guest-CPU context.
4422	* @param pcLoops Pointer to the number of executed loops.
4423	*/
4424	static int hmR0SvmRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4425	{
4426	uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4427	Assert(pcLoops);
4428	Assert(*pcLoops <= cMaxResumeLoops);
4429
4430	SVMTRANSIENT SvmTransient;
4431	SvmTransient.fUpdateTscOffsetting = true;
4432
4433	int rc = VERR_INTERNAL_ERROR_5;
4434	for (;;)
4435	{
4436	Assert(!HMR0SuspendPending());
4437	HMSVM_ASSERT_CPU_SAFE();
4438
4439	/* Preparatory work for running guest code, this may force us to return
4440	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4441	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4442	rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4443	if (rc != VINF_SUCCESS)
4444	break;
4445
4446	/*
4447	* No longjmps to ring-3 from this point on!!!
4448	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4449	* This also disables flushing of the R0-logger instance (if any).
4450	*/
4451	hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
4452	rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4453
4454	/* Restore any residual host-state and save any bits shared between host
4455	and guest into the guest-CPU state. Re-enables interrupts! */
4456	hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
4457
4458	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4459	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4460	{
4461	if (rc == VINF_SUCCESS)
4462	rc = VERR_SVM_INVALID_GUEST_STATE;
4463	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4464	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4465	break;
4466	}
4467
4468	/* Handle the #VMEXIT. */
4469	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4470	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4471	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4472	rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4473	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4474	if (rc != VINF_SUCCESS)
4475	break;
4476	if (++(*pcLoops) >= cMaxResumeLoops)
4477	{
4478	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4479	rc = VINF_EM_RAW_INTERRUPT;
4480	break;
4481	}
4482	}
4483
4484	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4485	return rc;
4486	}
4487
4488
4489	/**
4490	* Runs the guest code using AMD-V in single step mode.
4491	*
4492	* @returns VBox status code.
4493	* @param pVM The cross context VM structure.
4494	* @param pVCpu The cross context virtual CPU structure.
4495	* @param pCtx Pointer to the guest-CPU context.
4496	* @param pcLoops Pointer to the number of executed loops.
4497	*/
4498	static int hmR0SvmRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4499	{
4500	uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4501	Assert(pcLoops);
4502	Assert(*pcLoops <= cMaxResumeLoops);
4503
4504	SVMTRANSIENT SvmTransient;
4505	SvmTransient.fUpdateTscOffsetting = true;
4506
4507	uint16_t uCsStart = pCtx->cs.Sel;
4508	uint64_t uRipStart = pCtx->rip;
4509
4510	int rc = VERR_INTERNAL_ERROR_5;
4511	for (;;)
4512	{
4513	Assert(!HMR0SuspendPending());
4514	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
4515	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
4516	(unsigned)RTMpCpuId(), *pcLoops));
4517
4518	/* Preparatory work for running guest code, this may force us to return
4519	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4520	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4521	rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4522	if (rc != VINF_SUCCESS)
4523	break;
4524
4525	/*
4526	* No longjmps to ring-3 from this point on!!!
4527	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4528	* This also disables flushing of the R0-logger instance (if any).
4529	*/
4530	VMMRZCallRing3Disable(pVCpu);
4531	VMMRZCallRing3RemoveNotification(pVCpu);
4532	hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
4533
4534	rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4535
4536	/*
4537	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
4538	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
4539	*/
4540	hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
4541	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4542	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4543	{
4544	if (rc == VINF_SUCCESS)
4545	rc = VERR_SVM_INVALID_GUEST_STATE;
4546	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4547	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4548	return rc;
4549	}
4550
4551	/* Handle the #VMEXIT. */
4552	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4553	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4554	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4555	rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4556	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4557	if (rc != VINF_SUCCESS)
4558	break;
4559	if (++(*pcLoops) >= cMaxResumeLoops)
4560	{
4561	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4562	rc = VINF_EM_RAW_INTERRUPT;
4563	break;
4564	}
4565
4566	/*
4567	* Did the RIP change, if so, consider it a single step.
4568	* Otherwise, make sure one of the TFs gets set.
4569	*/
4570	if ( pCtx->rip != uRipStart
4571	\|\| pCtx->cs.Sel != uCsStart)
4572	{
4573	rc = VINF_EM_DBG_STEPPED;
4574	break;
4575	}
4576	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
4577	}
4578
4579	/*
4580	* Clear the X86_EFL_TF if necessary.
4581	*/
4582	if (pVCpu->hm.s.fClearTrapFlag)
4583	{
4584	pVCpu->hm.s.fClearTrapFlag = false;
4585	pCtx->eflags.Bits.u1TF = 0;
4586	}
4587
4588	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4589	return rc;
4590	}
4591
4592	#ifdef VBOX_WITH_NESTED_HWVIRT
4593	/**
4594	* Runs the nested-guest code using AMD-V.
4595	*
4596	* @returns VBox status code.
4597	* @param pVM The cross context VM structure.
4598	* @param pVCpu The cross context virtual CPU structure.
4599	* @param pCtx Pointer to the guest-CPU context.
4600	* @param pcLoops Pointer to the number of executed loops. If we're switching
4601	* from the guest-code execution loop to this nested-guest
4602	* execution loop pass the remainder value, else pass 0.
4603	*/
4604	static int hmR0SvmRunGuestCodeNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4605	{
4606	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4607	Assert(pcLoops);
4608	Assert(*pcLoops <= pVM->hm.s.cMaxResumeLoops);
4609
4610	SVMTRANSIENT SvmTransient;
4611	SvmTransient.fUpdateTscOffsetting = true;
4612
4613	int rc = VERR_INTERNAL_ERROR_4;
4614	for (;;)
4615	{
4616	Assert(!HMR0SuspendPending());
4617	HMSVM_ASSERT_CPU_SAFE();
4618
4619	/* Preparatory work for running nested-guest code, this may force us to return
4620	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4621	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4622	rc = hmR0SvmPreRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient);
4623	if (rc != VINF_SUCCESS)
4624	break;
4625
4626	/*
4627	* No longjmps to ring-3 from this point on!!!
4628	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4629	* This also disables flushing of the R0-logger instance (if any).
4630	*/
4631	hmR0SvmPreRunGuestCommittedNested(pVM, pVCpu, pCtx, &SvmTransient);
4632
4633	rc = hmR0SvmRunGuestNested(pVM, pVCpu, pCtx);
4634
4635	/* Restore any residual host-state and save any bits shared between host
4636	and guest into the guest-CPU state. Re-enables interrupts! */
4637	hmR0SvmPostRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient, rc);
4638
4639	/** @todo This needs some work... we probably should cause a \#VMEXIT on
4640	* SVM_EXIT_INVALID and handle rc != VINF_SUCCESS differently. */
4641	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4642	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4643	{
4644	if (rc == VINF_SUCCESS)
4645	rc = VERR_SVM_INVALID_GUEST_STATE;
4646	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4647	hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4648	break;
4649	}
4650
4651	/* Handle the #VMEXIT. */
4652	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4653	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4654	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
4655	rc = hmR0SvmHandleExitNested(pVCpu, pCtx, &SvmTransient);
4656	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4657	if (rc != VINF_SUCCESS)
4658	break;
4659	if (++(*pcLoops) >= pVM->hm.s.cMaxResumeLoops)
4660	{
4661	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4662	rc = VINF_EM_RAW_INTERRUPT;
4663	break;
4664	}
4665
4666	/** @todo handle single-stepping */
4667	}
4668
4669	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4670	return rc;
4671	}
4672	#endif
4673
4674
4675	/**
4676	* Runs the guest code using AMD-V.
4677	*
4678	* @returns Strict VBox status code.
4679	* @param pVM The cross context VM structure.
4680	* @param pVCpu The cross context virtual CPU structure.
4681	* @param pCtx Pointer to the guest-CPU context.
4682	*/
4683	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4684	{
4685	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4686	HMSVM_ASSERT_PREEMPT_SAFE();
4687	VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, pCtx);
4688
4689	uint32_t cLoops = 0;
4690	int rc;
4691	#ifdef VBOX_WITH_NESTED_HWVIRT
4692	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4693	#endif
4694	{
4695	if (!pVCpu->hm.s.fSingleInstruction)
4696	rc = hmR0SvmRunGuestCodeNormal(pVM, pVCpu, pCtx, &cLoops);
4697	else
4698	rc = hmR0SvmRunGuestCodeStep(pVM, pVCpu, pCtx, &cLoops);
4699	}
4700	#ifdef VBOX_WITH_NESTED_HWVIRT
4701	else
4702	{
4703	rc = VINF_SVM_VMRUN;
4704	}
4705
4706	/* Re-check the nested-guest condition here as we may be transitioning from the normal
4707	execution loop into the nested-guest, hence this is not placed in the 'else' part above. */
4708	if (rc == VINF_SVM_VMRUN)
4709	{
4710	rc = hmR0SvmRunGuestCodeNested(pVM, pVCpu, pCtx, &cLoops);
4711	if (rc == VINF_SVM_VMEXIT)
4712	rc = VINF_SUCCESS;
4713	}
4714	#endif
4715
4716	/* Fixup error codes. */
4717	if (rc == VERR_EM_INTERPRETER)
4718	rc = VINF_EM_RAW_EMULATE_INSTR;
4719	else if (rc == VINF_EM_RESET)
4720	rc = VINF_EM_TRIPLE_FAULT;
4721
4722	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4723	rc = hmR0SvmExitToRing3(pVM, pVCpu, pCtx, rc);
4724	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
4725	return rc;
4726	}
4727
4728
4729	#ifdef VBOX_WITH_NESTED_HWVIRT
4730	/**
4731	* Determines whether an IOIO intercept is active for the nested-guest or not.
4732	*
4733	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4734	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4735	*/
4736	static bool hmR0SvmIsIoInterceptActive(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4737	{
4738	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4739	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4740	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4741	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4742	const uint8_t iEffSeg = pIoExitInfo->n.u3SEG;
4743	const bool fRep = pIoExitInfo->n.u1REP;
4744	const bool fStrIo = pIoExitInfo->n.u1STR;
4745
4746	return HMSvmIsIOInterceptActive(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4747	NULL /* pIoExitInfo */);
4748	}
4749
4750
4751	/**
4752	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4753	* SVM_EXIT_INVALID).
4754	*
4755	* @returns VBox status code (informational status codes included).
4756	* @param pVCpu The cross context virtual CPU structure.
4757	* @param pCtx Pointer to the guest-CPU context.
4758	* @param pSvmTransient Pointer to the SVM transient structure.
4759	*/
4760	static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4761	{
4762	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4763	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4764	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4765
4766	#define HM_SVM_VMEXIT_NESTED(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4767	VBOXSTRICTRC_TODO(IEMExecSvmVmexit(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2))
4768
4769	/*
4770	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected
4771	* by the nested-guest. If it isn't, it should be handled by the (outer) guest.
4772	*/
4773	PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4774	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4775	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4776	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4777	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4778
4779	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4780	switch (uExitCode)
4781	{
4782	case SVM_EXIT_CPUID:
4783	{
4784	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4785	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4786	return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
4787	}
4788
4789	case SVM_EXIT_RDTSC:
4790	{
4791	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4792	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4793	return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
4794	}
4795
4796	case SVM_EXIT_RDTSCP:
4797	{
4798	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4799	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4800	return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
4801	}
4802
4803
4804	case SVM_EXIT_MONITOR:
4805	{
4806	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4807	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4808	return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
4809	}
4810
4811	case SVM_EXIT_MWAIT:
4812	{
4813	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4814	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4815	return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
4816	}
4817
4818	case SVM_EXIT_HLT:
4819	{
4820	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4821	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4822	return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
4823	}
4824
4825	case SVM_EXIT_MSR:
4826	{
4827	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4828	{
4829	uint32_t const idMsr = pCtx->ecx;
4830	uint16_t offMsrpm;
4831	uint32_t uMsrpmBit;
4832	int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4833	if (RT_SUCCESS(rc))
4834	{
4835	void const *pvMsrBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4836	bool const fInterceptRead = ASMBitTest(pvMsrBitmap, (offMsrpm << 3) + uMsrpmBit);
4837	bool const fInterceptWrite = ASMBitTest(pvMsrBitmap, (offMsrpm << 3) + uMsrpmBit + 1);
4838
4839	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4840	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4841	{
4842	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4843	}
4844	}
4845	else
4846	{
4847	/*
4848	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4849	* See AMD-V spec. "15.11 MSR Intercepts".
4850	*/
4851	Assert(rc == VERR_OUT_OF_RANGE);
4852	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4853	}
4854	}
4855	return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
4856	}
4857
4858	case SVM_EXIT_IOIO:
4859	{
4860	/*
4861	* Figure out if the IO port access is intercepted by the nested-guest.
4862	*/
4863	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4864	{
4865	void *pvIoBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvIoBitmap);
4866	SVMIOIOEXITINFO IoExitInfo;
4867	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4868	bool const fIntercept = hmR0SvmIsIoInterceptActive(pvIoBitmap, &IoExitInfo);
4869	if (fIntercept)
4870	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4871	}
4872	return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
4873	}
4874
4875	case SVM_EXIT_EXCEPTION_14: /* X86_XCPT_PF */
4876	{
4877	PVM pVM = pVCpu->CTX_SUFF(pVM);
4878	if (pVM->hm.s.fNestedPaging)
4879	{
4880	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4881	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4882
4883	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4884	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4885	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4886
4887	/* If the nested-guest is not intercepting #PFs, forward the #PF to the nested-guest. */
4888	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
4889	return VINF_SUCCESS;
4890	}
4891	return hmR0SvmExitXcptPFNested(pVCpu, pCtx,pSvmTransient);
4892	}
4893
4894	case SVM_EXIT_EXCEPTION_7: /* X86_XCPT_NM */
4895	{
4896	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_NM))
4897	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4898	hmR0SvmSetPendingXcptNM(pVCpu);
4899	return VINF_SUCCESS;
4900	}
4901
4902	case SVM_EXIT_EXCEPTION_6: /* X86_XCPT_UD */
4903	{
4904	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4905	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4906	hmR0SvmSetPendingXcptUD(pVCpu);
4907	return VINF_SUCCESS;
4908	}
4909
4910	case SVM_EXIT_EXCEPTION_16: /* X86_XCPT_MF */
4911	{
4912	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4913	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4914	hmR0SvmSetPendingXcptMF(pVCpu);
4915	return VINF_SUCCESS;
4916	}
4917
4918	case SVM_EXIT_EXCEPTION_1: /* X86_XCPT_DB */
4919	{
4920	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4921	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4922	return hmR0SvmNestedExitXcptDB(pVCpu, pCtx, pSvmTransient);
4923	}
4924
4925	case SVM_EXIT_EXCEPTION_17: /* X86_XCPT_AC */
4926	{
4927	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4928	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4929	return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
4930	}
4931
4932	case SVM_EXIT_EXCEPTION_3: /* X86_XCPT_BP */
4933	{
4934	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
4935	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4936	return hmR0SvmNestedExitXcptBP(pVCpu, pCtx, pSvmTransient);
4937	}
4938
4939	case SVM_EXIT_READ_CR0:
4940	case SVM_EXIT_READ_CR3:
4941	case SVM_EXIT_READ_CR4:
4942	{
4943	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
4944	if (HMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
4945	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4946	return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
4947	}
4948
4949	case SVM_EXIT_WRITE_CR0:
4950	case SVM_EXIT_WRITE_CR3:
4951	case SVM_EXIT_WRITE_CR4:
4952	case SVM_EXIT_WRITE_CR8: /** @todo Shouldn't writes to CR8 go to V_TPR instead since we run with V_INTR_MASKING set?? */
4953	{
4954	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
4955	Log4(("hmR0SvmHandleExitNested: Write CRx: u16InterceptWrCRx=%#x u64ExitCode=%#RX64 %#x\n",
4956	pVmcbNstGstCtrl->u16InterceptWrCRx, pSvmTransient->u64ExitCode, uCr));
4957
4958	if (HMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
4959	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4960	return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
4961	}
4962
4963	case SVM_EXIT_PAUSE:
4964	{
4965	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
4966	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4967	return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
4968	}
4969
4970	case SVM_EXIT_VINTR:
4971	{
4972	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
4973	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4974	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
4975	}
4976
4977	case SVM_EXIT_INTR:
4978	case SVM_EXIT_NMI:
4979	case SVM_EXIT_SMI:
4980	{
4981	/*
4982	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
4983	*
4984	* Although we don't intercept SMIs, the nested-guest might. Therefore, we
4985	* might get an SMI #VMEXIT here so simply ignore rather than causing a
4986	* corresponding nested-guest #VMEXIT.
4987	*/
4988	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
4989	}
4990
4991	case SVM_EXIT_FERR_FREEZE:
4992	{
4993	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
4994	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4995	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
4996	}
4997
4998	case SVM_EXIT_INVLPG:
4999	{
5000	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5001	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5002	return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5003	}
5004
5005	case SVM_EXIT_WBINVD:
5006	{
5007	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5008	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5009	return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5010	}
5011
5012	case SVM_EXIT_INVD:
5013	{
5014	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5015	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5016	return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5017	}
5018
5019	case SVM_EXIT_RDPMC:
5020	{
5021	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5022	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5023	return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5024	}
5025
5026	default:
5027	{
5028	switch (uExitCode)
5029	{
5030	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5031	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5032	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5033	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5034	{
5035	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5036	if (HMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5037	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5038	return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5039	}
5040
5041	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5042	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5043	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5044	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5045	{
5046	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5047	if (HMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5048	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5049	return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5050	}
5051
5052	/* The exceptions not handled here are already handled individually above (as they occur more frequently). */
5053	case SVM_EXIT_EXCEPTION_0: /case SVM_EXIT_EXCEPTION_1:/ case SVM_EXIT_EXCEPTION_2:
5054	/case SVM_EXIT_EXCEPTION_3:/ case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5:
5055	/case SVM_EXIT_EXCEPTION_6:/ /case SVM_EXIT_EXCEPTION_7:/ case SVM_EXIT_EXCEPTION_8:
5056	case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11:
5057	case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13: /case SVM_EXIT_EXCEPTION_14:/
5058	case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: /case SVM_EXIT_EXCEPTION_17:/
5059	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_20:
5060	case SVM_EXIT_EXCEPTION_21: case SVM_EXIT_EXCEPTION_22: case SVM_EXIT_EXCEPTION_23:
5061	case SVM_EXIT_EXCEPTION_24: case SVM_EXIT_EXCEPTION_25: case SVM_EXIT_EXCEPTION_26:
5062	case SVM_EXIT_EXCEPTION_27: case SVM_EXIT_EXCEPTION_28: case SVM_EXIT_EXCEPTION_29:
5063	case SVM_EXIT_EXCEPTION_30: case SVM_EXIT_EXCEPTION_31:
5064	{
5065	uint8_t const uVector = uExitCode - SVM_EXIT_EXCEPTION_0;
5066	if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5067	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5068	return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5069	}
5070
5071	case SVM_EXIT_XSETBV:
5072	{
5073	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5074	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5075	return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5076	}
5077
5078	case SVM_EXIT_TASK_SWITCH:
5079	{
5080	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5081	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5082	return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5083	}
5084
5085	case SVM_EXIT_IRET:
5086	{
5087	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5088	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5089	return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5090	}
5091
5092	case SVM_EXIT_SHUTDOWN:
5093	{
5094	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5095	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5096	return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5097	}
5098
5099	case SVM_EXIT_VMMCALL:
5100	{
5101	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5102	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5103	return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5104	}
5105
5106	case SVM_EXIT_CLGI:
5107	{
5108	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5109	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5110	return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5111	}
5112
5113	case SVM_EXIT_STGI:
5114	{
5115	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5116	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5117	return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5118	}
5119
5120	case SVM_EXIT_VMLOAD:
5121	{
5122	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5123	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5124	return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5125	}
5126
5127	case SVM_EXIT_VMSAVE:
5128	{
5129	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5130	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5131	return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5132	}
5133
5134	case SVM_EXIT_INVLPGA:
5135	{
5136	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5137	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5138	return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5139	}
5140
5141	case SVM_EXIT_VMRUN:
5142	{
5143	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5144	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5145	return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5146	}
5147
5148	case SVM_EXIT_RSM:
5149	{
5150	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5151	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5152	hmR0SvmSetPendingXcptUD(pVCpu);
5153	return VINF_SUCCESS;
5154	}
5155
5156	case SVM_EXIT_SKINIT:
5157	{
5158	if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5159	return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5160	hmR0SvmSetPendingXcptUD(pVCpu);
5161	return VINF_SUCCESS;
5162	}
5163
5164	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5165	case SVM_EXIT_NPF: /* We don't yet support nested-paging for nested-guests, so this should never happen. */
5166	{
5167	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5168	}
5169
5170	default:
5171	{
5172	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5173	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5174	return VERR_SVM_UNKNOWN_EXIT;
5175	}
5176	}
5177	}
5178	}
5179	/* not reached */
5180
5181	#undef HM_SVM_VMEXIT_NESTED
5182	}
5183	#endif
5184
5185
5186	/**
5187	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5188	*
5189	* @returns VBox status code (informational status codes included).
5190	* @param pVCpu The cross context virtual CPU structure.
5191	* @param pCtx Pointer to the guest-CPU context.
5192	* @param pSvmTransient Pointer to the SVM transient structure.
5193	*/
5194	static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5195	{
5196	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5197	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5198
5199	/*
5200	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs for most guests under
5201	* normal workloads (for some definition of "normal").
5202	*/
5203	uint32_t u32ExitCode = pSvmTransient->u64ExitCode;
5204	switch (pSvmTransient->u64ExitCode)
5205	{
5206	case SVM_EXIT_NPF:
5207	return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
5208
5209	case SVM_EXIT_IOIO:
5210	return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
5211
5212	case SVM_EXIT_RDTSC:
5213	return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
5214
5215	case SVM_EXIT_RDTSCP:
5216	return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
5217
5218	case SVM_EXIT_CPUID:
5219	return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
5220
5221	case SVM_EXIT_EXCEPTION_14: /* X86_XCPT_PF */
5222	return hmR0SvmExitXcptPF(pVCpu, pCtx, pSvmTransient);
5223
5224	case SVM_EXIT_EXCEPTION_7: /* X86_XCPT_NM */
5225	return hmR0SvmExitXcptNM(pVCpu, pCtx, pSvmTransient);
5226
5227	case SVM_EXIT_EXCEPTION_6: /* X86_XCPT_UD */
5228	return hmR0SvmExitXcptUD(pVCpu, pCtx, pSvmTransient);
5229
5230	case SVM_EXIT_EXCEPTION_16: /* X86_XCPT_MF */
5231	return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
5232
5233	case SVM_EXIT_EXCEPTION_1: /* X86_XCPT_DB */
5234	return hmR0SvmExitXcptDB(pVCpu, pCtx, pSvmTransient);
5235
5236	case SVM_EXIT_EXCEPTION_17: /* X86_XCPT_AC */
5237	return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5238
5239	case SVM_EXIT_EXCEPTION_3: /* X86_XCPT_BP */
5240	return hmR0SvmExitXcptBP(pVCpu, pCtx, pSvmTransient);
5241
5242	case SVM_EXIT_MONITOR:
5243	return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
5244
5245	case SVM_EXIT_MWAIT:
5246	return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
5247
5248	case SVM_EXIT_HLT:
5249	return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
5250
5251	case SVM_EXIT_READ_CR0:
5252	case SVM_EXIT_READ_CR3:
5253	case SVM_EXIT_READ_CR4:
5254	return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5255
5256	case SVM_EXIT_WRITE_CR0:
5257	case SVM_EXIT_WRITE_CR3:
5258	case SVM_EXIT_WRITE_CR4:
5259	case SVM_EXIT_WRITE_CR8:
5260	return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5261
5262	case SVM_EXIT_PAUSE:
5263	return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5264
5265	case SVM_EXIT_VMMCALL:
5266	return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5267
5268	case SVM_EXIT_VINTR:
5269	return hmR0SvmExitVIntr(pVCpu, pCtx, pSvmTransient);
5270
5271	case SVM_EXIT_INTR:
5272	case SVM_EXIT_FERR_FREEZE:
5273	case SVM_EXIT_NMI:
5274	return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5275
5276	case SVM_EXIT_MSR:
5277	return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
5278
5279	case SVM_EXIT_INVLPG:
5280	return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5281
5282	case SVM_EXIT_WBINVD:
5283	return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5284
5285	case SVM_EXIT_INVD:
5286	return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5287
5288	case SVM_EXIT_RDPMC:
5289	return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5290
5291	default:
5292	{
5293	switch (pSvmTransient->u64ExitCode)
5294	{
5295	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5296	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5297	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5298	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5299	return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5300
5301	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5302	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5303	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5304	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5305	return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5306
5307	case SVM_EXIT_XSETBV:
5308	return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5309
5310	case SVM_EXIT_TASK_SWITCH:
5311	return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5312
5313	case SVM_EXIT_IRET:
5314	return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5315
5316	case SVM_EXIT_SHUTDOWN:
5317	return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5318
5319	case SVM_EXIT_SMI:
5320	case SVM_EXIT_INIT:
5321	{
5322	/*
5323	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5324	* If it ever does, we want to know about it so log the exit code and bail.
5325	*/
5326	return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5327	}
5328
5329	#ifdef VBOX_WITH_NESTED_HWVIRT
5330	case SVM_EXIT_CLGI: return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5331	case SVM_EXIT_STGI: return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5332	case SVM_EXIT_VMLOAD: return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5333	case SVM_EXIT_VMSAVE: return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5334	case SVM_EXIT_INVLPGA: return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5335	case SVM_EXIT_VMRUN: return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5336	#else
5337	case SVM_EXIT_CLGI:
5338	case SVM_EXIT_STGI:
5339	case SVM_EXIT_VMLOAD:
5340	case SVM_EXIT_VMSAVE:
5341	case SVM_EXIT_INVLPGA:
5342	case SVM_EXIT_VMRUN:
5343	#endif
5344	case SVM_EXIT_RSM:
5345	case SVM_EXIT_SKINIT:
5346	{
5347	hmR0SvmSetPendingXcptUD(pVCpu);
5348	return VINF_SUCCESS;
5349	}
5350
5351	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5352	case SVM_EXIT_EXCEPTION_0: /* X86_XCPT_DE */
5353	/* SVM_EXIT_EXCEPTION_1: / / X86_XCPT_DB - Handled above. */
5354	case SVM_EXIT_EXCEPTION_2: /* X86_XCPT_NMI */
5355	/* SVM_EXIT_EXCEPTION_3: / / X86_XCPT_BP - Handled above. */
5356	case SVM_EXIT_EXCEPTION_4: /* X86_XCPT_OF */
5357	case SVM_EXIT_EXCEPTION_5: /* X86_XCPT_BR */
5358	/* SVM_EXIT_EXCEPTION_6: / / X86_XCPT_UD - Handled above. */
5359	/* SVM_EXIT_EXCEPTION_7: / / X86_XCPT_NM - Handled above. */
5360	case SVM_EXIT_EXCEPTION_8: /* X86_XCPT_DF */
5361	case SVM_EXIT_EXCEPTION_9: /* X86_XCPT_CO_SEG_OVERRUN */
5362	case SVM_EXIT_EXCEPTION_10: /* X86_XCPT_TS */
5363	case SVM_EXIT_EXCEPTION_11: /* X86_XCPT_NP */
5364	case SVM_EXIT_EXCEPTION_12: /* X86_XCPT_SS */
5365	case SVM_EXIT_EXCEPTION_13: /* X86_XCPT_GP */
5366	/* SVM_EXIT_EXCEPTION_14: / / X86_XCPT_PF - Handled above. */
5367	case SVM_EXIT_EXCEPTION_15: /* Reserved. */
5368	/* SVM_EXIT_EXCEPTION_16: / / X86_XCPT_MF - Handled above. */
5369	/* SVM_EXIT_EXCEPTION_17: / / X86_XCPT_AC - Handled above. */
5370	case SVM_EXIT_EXCEPTION_18: /* X86_XCPT_MC */
5371	case SVM_EXIT_EXCEPTION_19: /* X86_XCPT_XF */
5372	case SVM_EXIT_EXCEPTION_20: case SVM_EXIT_EXCEPTION_21: case SVM_EXIT_EXCEPTION_22:
5373	case SVM_EXIT_EXCEPTION_23: case SVM_EXIT_EXCEPTION_24: case SVM_EXIT_EXCEPTION_25:
5374	case SVM_EXIT_EXCEPTION_26: case SVM_EXIT_EXCEPTION_27: case SVM_EXIT_EXCEPTION_28:
5375	case SVM_EXIT_EXCEPTION_29: case SVM_EXIT_EXCEPTION_30: case SVM_EXIT_EXCEPTION_31:
5376	{
5377	/** @todo r=ramshankar; We should be doing
5378	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY here! */
5379	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
5380	SVMEVENT Event;
5381	Event.u = 0;
5382	Event.n.u1Valid = 1;
5383	Event.n.u3Type = SVM_EVENT_EXCEPTION;
5384	Event.n.u8Vector = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0;
5385
5386	switch (Event.n.u8Vector)
5387	{
5388	case X86_XCPT_DE:
5389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
5390	break;
5391
5392	case X86_XCPT_NP:
5393	Event.n.u1ErrorCodeValid = 1;
5394	Event.n.u32ErrorCode = pVmcb->ctrl.u64ExitInfo1;
5395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
5396	break;
5397
5398	case X86_XCPT_SS:
5399	Event.n.u1ErrorCodeValid = 1;
5400	Event.n.u32ErrorCode = pVmcb->ctrl.u64ExitInfo1;
5401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
5402	break;
5403
5404	case X86_XCPT_GP:
5405	Event.n.u1ErrorCodeValid = 1;
5406	Event.n.u32ErrorCode = pVmcb->ctrl.u64ExitInfo1;
5407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
5408	break;
5409
5410	default:
5411	AssertMsgFailed(("hmR0SvmHandleExit: Unexpected exit caused by exception %#x\n", Event.n.u8Vector));
5412	pVCpu->hm.s.u32HMError = Event.n.u8Vector;
5413	return VERR_SVM_UNEXPECTED_XCPT_EXIT;
5414	}
5415
5416	Log4(("#Xcpt: Vector=%#x at CS:RIP=%04x:%RGv\n", Event.n.u8Vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
5417	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
5418	return VINF_SUCCESS;
5419	}
5420	#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5421
5422	default:
5423	{
5424	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#x\n", u32ExitCode));
5425	pVCpu->hm.s.u32HMError = u32ExitCode;
5426	return VERR_SVM_UNKNOWN_EXIT;
5427	}
5428	}
5429	}
5430	}
5431	/* not reached */
5432	}
5433
5434
5435	#ifdef DEBUG
5436	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5437	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5438	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5439
5440	# define HMSVM_ASSERT_PREEMPT_CPUID() \
5441	do \
5442	{ \
5443	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5444	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5445	} while (0)
5446
5447	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() \
5448	do { \
5449	AssertPtr(pVCpu); \
5450	AssertPtr(pCtx); \
5451	AssertPtr(pSvmTransient); \
5452	Assert(ASMIntAreEnabled()); \
5453	HMSVM_ASSERT_PREEMPT_SAFE(); \
5454	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5455	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
5456	HMSVM_ASSERT_PREEMPT_SAFE(); \
5457	if (VMMR0IsLogFlushDisabled(pVCpu)) \
5458	HMSVM_ASSERT_PREEMPT_CPUID(); \
5459	} while (0)
5460	#else /* Release builds */
5461	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() do { NOREF(pVCpu); NOREF(pCtx); NOREF(pSvmTransient); } while (0)
5462	#endif
5463
5464
5465	/**
5466	* Worker for hmR0SvmInterpretInvlpg().
5467	*
5468	* @return VBox status code.
5469	* @param pVCpu The cross context virtual CPU structure.
5470	* @param pCpu Pointer to the disassembler state.
5471	* @param pCtx The guest CPU context.
5472	*/
5473	static int hmR0SvmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTX pCtx)
5474	{
5475	DISQPVPARAMVAL Param1;
5476	RTGCPTR GCPtrPage;
5477
5478	int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), pCpu, &pCpu->Param1, &Param1, DISQPVWHICH_SRC);
5479	if (RT_FAILURE(rc))
5480	return VERR_EM_INTERPRETER;
5481
5482	if ( Param1.type == DISQPV_TYPE_IMMEDIATE
5483	\|\| Param1.type == DISQPV_TYPE_ADDRESS)
5484	{
5485	if (!(Param1.flags & (DISQPV_FLAG_32 \| DISQPV_FLAG_64)))
5486	return VERR_EM_INTERPRETER;
5487
5488	GCPtrPage = Param1.val.val64;
5489	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), GCPtrPage);
5490	rc = VBOXSTRICTRC_VAL(rc2);
5491	}
5492	else
5493	{
5494	Log4(("hmR0SvmInterpretInvlPgEx invalid parameter type %#x\n", Param1.type));
5495	rc = VERR_EM_INTERPRETER;
5496	}
5497
5498	return rc;
5499	}
5500
5501
5502	/**
5503	* Interprets INVLPG.
5504	*
5505	* @returns VBox status code.
5506	* @retval VINF_* Scheduling instructions.
5507	* @retval VERR_EM_INTERPRETER Something we can't cope with.
5508	* @retval VERR_* Fatal errors.
5509	*
5510	* @param pVM The cross context VM structure.
5511	* @param pVCpu The cross context virtual CPU structure.
5512	* @param pCtx The guest CPU context.
5513	*
5514	* @remarks Updates the RIP if the instruction was executed successfully.
5515	*/
5516	static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5517	{
5518	/* Only allow 32 & 64 bit code. */
5519	if (CPUMGetGuestCodeBits(pVCpu) != 16)
5520	{
5521	PDISSTATE pDis = &pVCpu->hm.s.DisState;
5522	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
5523	if ( RT_SUCCESS(rc)
5524	&& pDis->pCurInstr->uOpcode == OP_INVLPG)
5525	{
5526	rc = hmR0SvmInterpretInvlPgEx(pVCpu, pDis, pCtx);
5527	if (RT_SUCCESS(rc))
5528	pCtx->rip += pDis->cbInstr;
5529	return rc;
5530	}
5531	else
5532	Log4(("hmR0SvmInterpretInvlpg: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
5533	}
5534	return VERR_EM_INTERPRETER;
5535	}
5536
5537
5538	#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5539	/**
5540	* Gets the IEM exception flags for the specified SVM event.
5541	*
5542	* @returns The IEM exception flags.
5543	* @param pEvent Pointer to the SVM event.
5544	*
5545	* @remarks This function currently only constructs flags required for
5546	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5547	* and CR2 aspects of an exception are not included).
5548	*/
5549	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
5550	{
5551	uint8_t const uEventType = pEvent->n.u3Type;
5552	uint32_t fIemXcptFlags;
5553	switch (uEventType)
5554	{
5555	case SVM_EVENT_EXCEPTION:
5556	/*
5557	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
5558	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
5559	*/
5560	if (pEvent->n.u8Vector == X86_XCPT_BP)
5561	{
5562	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
5563	break;
5564	}
5565	if (pEvent->n.u8Vector == X86_XCPT_OF)
5566	{
5567	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
5568	break;
5569	}
5570	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
5571	RT_FALL_THRU();
5572	case SVM_EVENT_NMI:
5573	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5574	break;
5575
5576	case SVM_EVENT_EXTERNAL_IRQ:
5577	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5578	break;
5579
5580	case SVM_EVENT_SOFTWARE_INT:
5581	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5582	break;
5583
5584	default:
5585	fIemXcptFlags = 0;
5586	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
5587	break;
5588	}
5589	return fIemXcptFlags;
5590	}
5591
5592	#else
5593	/**
5594	* Determines if an exception is a contributory exception.
5595	*
5596	* Contributory exceptions are ones which can cause double-faults unless the
5597	* original exception was a benign exception. Page-fault is intentionally not
5598	* included here as it's a conditional contributory exception.
5599	*
5600	* @returns @c true if the exception is contributory, @c false otherwise.
5601	* @param uVector The exception vector.
5602	*/
5603	DECLINLINE(bool) hmR0SvmIsContributoryXcpt(const uint32_t uVector)
5604	{
5605	switch (uVector)
5606	{
5607	case X86_XCPT_GP:
5608	case X86_XCPT_SS:
5609	case X86_XCPT_NP:
5610	case X86_XCPT_TS:
5611	case X86_XCPT_DE:
5612	return true;
5613	default:
5614	break;
5615	}
5616	return false;
5617	}
5618	#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5619
5620
5621	/**
5622	* Handle a condition that occurred while delivering an event through the guest
5623	* IDT.
5624	*
5625	* @returns VBox status code (informational error codes included).
5626	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
5627	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5628	* continue execution of the guest which will delivery the \#DF.
5629	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5630	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5631	*
5632	* @param pVCpu The cross context virtual CPU structure.
5633	* @param pCtx Pointer to the guest-CPU context.
5634	* @param pSvmTransient Pointer to the SVM transient structure.
5635	*
5636	* @remarks No-long-jump zone!!!
5637	*/
5638	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5639	{
5640	int rc = VINF_SUCCESS;
5641	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5642
5643	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
5644	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
5645	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
5646
5647	/* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code". The EXITINTINFO (if valid) contains the prior exception (IDT vector)
5648	* that was trying to be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector). */
5649	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
5650	{
5651	#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5652	IEMXCPTRAISE enmRaise;
5653	IEMXCPTRAISEINFO fRaiseInfo;
5654	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31;
5655	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5656	if (fExitIsHwXcpt)
5657	{
5658	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0;
5659	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
5660	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5661	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5662	}
5663	else
5664	{
5665	/*
5666	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF) then we
5667	* end up here.
5668	*
5669	* If the event was:
5670	* - a software interrupt, we can re-execute the instruction which will regenerate
5671	* the event.
5672	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
5673	* - a hardware exception or external interrupt, we re-inject it.
5674	*/
5675	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5676	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
5677	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5678	else
5679	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5680	}
5681
5682	switch (enmRaise)
5683	{
5684	case IEMXCPTRAISE_CURRENT_XCPT:
5685	case IEMXCPTRAISE_PREV_EVENT:
5686	{
5687	/* For software interrupts, we shall re-execute the instruction. */
5688	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
5689	{
5690	RTGCUINTPTR GCPtrFaultAddress = 0;
5691
5692	/* If we are re-injecting an NMI, clear NMI blocking. */
5693	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5694	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5695
5696	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
5697	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5698	pSvmTransient->fVectoringPF = true;
5699	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
5700	&& uIdtVector == X86_XCPT_PF)
5701	{
5702	/*
5703	* If the previous exception was a #PF, we need to recover the CR2 value.
5704	* This can't happen with shadow paging.
5705	*/
5706	GCPtrFaultAddress = pCtx->cr2;
5707	}
5708
5709	/*
5710	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
5711	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
5712	*/
5713	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5714	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5715	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
5716
5717	Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
5718	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
5719	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
5720	}
5721	break;
5722	}
5723
5724	case IEMXCPTRAISE_REEXEC_INSTR:
5725	{
5726	Assert(rc == VINF_SUCCESS);
5727	break;
5728	}
5729
5730	case IEMXCPTRAISE_DOUBLE_FAULT:
5731	{
5732	/*
5733	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5734	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5735	*/
5736	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5737	{
5738	pSvmTransient->fVectoringDoublePF = true;
5739	Assert(rc == VINF_SUCCESS);
5740	}
5741	else
5742	{
5743	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5744	hmR0SvmSetPendingXcptDF(pVCpu);
5745	rc = VINF_HM_DOUBLE_FAULT;
5746	}
5747	break;
5748	}
5749
5750	case IEMXCPTRAISE_TRIPLE_FAULT:
5751	{
5752	rc = VINF_EM_RESET;
5753	break;
5754	}
5755
5756	case IEMXCPTRAISE_CPU_HANG:
5757	{
5758	rc = VERR_EM_GUEST_CPU_HANG;
5759	break;
5760	}
5761
5762	default:
5763	{
5764	AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
5765	rc = VERR_SVM_IPE_2;
5766	break;
5767	}
5768	}
5769	#else
5770	uint8_t uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5771
5772	typedef enum
5773	{
5774	SVMREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5775	SVMREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5776	SVMREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5777	SVMREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5778	SVMREFLECTXCPT_NONE /* Nothing to reflect. */
5779	} SVMREFLECTXCPT;
5780
5781	SVMREFLECTXCPT enmReflect = SVMREFLECTXCPT_NONE;
5782	bool fReflectingNmi = false;
5783	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION)
5784	{
5785	if (pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31)
5786	{
5787	uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0);
5788
5789	#ifdef VBOX_STRICT
5790	if ( hmR0SvmIsContributoryXcpt(uIdtVector)
5791	&& uExitVector == X86_XCPT_PF)
5792	{
5793	Log4(("IDT: Contributory #PF idCpu=%u uCR2=%#RX64\n", pVCpu->idCpu, pCtx->cr2));
5794	}
5795	#endif
5796
5797	if ( uIdtVector == X86_XCPT_BP
5798	\|\| uIdtVector == X86_XCPT_OF)
5799	{
5800	/* Ignore INT3/INTO, just re-execute. See @bugref{8357}. */
5801	}
5802	else if ( uExitVector == X86_XCPT_PF
5803	&& uIdtVector == X86_XCPT_PF)
5804	{
5805	pSvmTransient->fVectoringDoublePF = true;
5806	Log4(("IDT: Vectoring double #PF uCR2=%#RX64\n", pCtx->cr2));
5807	}
5808	else if ( uExitVector == X86_XCPT_AC
5809	&& uIdtVector == X86_XCPT_AC)
5810	{
5811	enmReflect = SVMREFLECTXCPT_HANG;
5812	Log4(("IDT: Nested #AC - Bad guest\n"));
5813	}
5814	else if ( (pVmcb->ctrl.u32InterceptXcpt & HMSVM_CONTRIBUTORY_XCPT_MASK)
5815	&& hmR0SvmIsContributoryXcpt(uExitVector)
5816	&& ( hmR0SvmIsContributoryXcpt(uIdtVector)
5817	\|\| uIdtVector == X86_XCPT_PF))
5818	{
5819	enmReflect = SVMREFLECTXCPT_DF;
5820	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5821	uIdtVector, uExitVector));
5822	}
5823	else if (uIdtVector == X86_XCPT_DF)
5824	{
5825	enmReflect = SVMREFLECTXCPT_TF;
5826	Log4(("IDT: Pending vectoring triple-fault %#RX64 uIdtVector=%#x uExitVector=%#x\n",
5827	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5828	}
5829	else
5830	enmReflect = SVMREFLECTXCPT_XCPT;
5831	}
5832	else
5833	{
5834	/*
5835	* If event delivery caused an #VMEXIT that is not an exception (e.g. #NPF) then reflect the original
5836	* exception to the guest after handling the #VMEXIT.
5837	*/
5838	enmReflect = SVMREFLECTXCPT_XCPT;
5839	}
5840	}
5841	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXTERNAL_IRQ
5842	\|\| pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5843	{
5844	enmReflect = SVMREFLECTXCPT_XCPT;
5845	fReflectingNmi = RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI);
5846
5847	if (pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31)
5848	{
5849	uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0);
5850	if (uExitVector == X86_XCPT_PF)
5851	{
5852	pSvmTransient->fVectoringPF = true;
5853	Log4(("IDT: Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pCtx->cr2));
5854	}
5855	}
5856	}
5857	/* else: Ignore software interrupts (INT n) as they reoccur when restarting the instruction. */
5858
5859	switch (enmReflect)
5860	{
5861	case SVMREFLECTXCPT_XCPT:
5862	{
5863	/* If we are re-injecting the NMI, clear NMI blocking. */
5864	if (fReflectingNmi)
5865	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5866
5867	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5868	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5869	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, 0 /* GCPtrFaultAddress */);
5870
5871	/* If uExitVector is #PF, CR2 value will be updated from the VMCB if it's a guest #PF. See hmR0SvmExitXcptPF(). */
5872	Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32\n", pVmcb->ctrl.ExitIntInfo.u,
5873	!!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid, pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
5874	break;
5875	}
5876
5877	case SVMREFLECTXCPT_DF:
5878	{
5879	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5880	hmR0SvmSetPendingXcptDF(pVCpu);
5881	rc = VINF_HM_DOUBLE_FAULT;
5882	break;
5883	}
5884
5885	case SVMREFLECTXCPT_TF:
5886	{
5887	rc = VINF_EM_RESET;
5888	break;
5889	}
5890
5891	case SVMREFLECTXCPT_HANG:
5892	{
5893	rc = VERR_EM_GUEST_CPU_HANG;
5894	break;
5895	}
5896
5897	default:
5898	Assert(rc == VINF_SUCCESS);
5899	break;
5900	}
5901	#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5902	}
5903	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
5904	NOREF(pCtx);
5905	return rc;
5906	}
5907
5908
5909	/**
5910	* Advances the guest RIP making use of the CPU's NRIP_SAVE feature if
5911	* supported, otherwise advances the RIP by the number of bytes specified in
5912	* @a cb.
5913	*
5914	* @param pVCpu The cross context virtual CPU structure.
5915	* @param pCtx Pointer to the guest-CPU context.
5916	* @param cb RIP increment value in bytes.
5917	*
5918	* @remarks Use this function only from \#VMEXIT's where the NRIP value is valid
5919	* when NRIP_SAVE is supported by the CPU, otherwise use
5920	* hmR0SvmAdvanceRipDumb!
5921	*/
5922	DECLINLINE(void) hmR0SvmAdvanceRipHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
5923	{
5924	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
5925	if (fSupportsNextRipSave)
5926	{
5927	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5928	Assert(pVmcb->ctrl.u64NextRIP);
5929	AssertRelease(pVmcb->ctrl.u64NextRIP - pCtx->rip == cb); /* temporary, remove later */
5930	pCtx->rip = pVmcb->ctrl.u64NextRIP;
5931	}
5932	else
5933	pCtx->rip += cb;
5934
5935	HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
5936	}
5937
5938
5939	#ifdef VBOX_WITH_NESTED_HWVIRT
5940	/**
5941	* Gets the length of the current instruction if the CPU supports the NRIP_SAVE
5942	* feature. Otherwise, returns the value in @a cbLikely.
5943	*
5944	* @param pVCpu The cross context virtual CPU structure.
5945	* @param pCtx Pointer to the guest-CPU context.
5946	* @param cbLikely The likely instruction length.
5947	*/
5948	DECLINLINE(uint8_t) hmR0SvmGetInstrLengthHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint8_t cbLikely)
5949	{
5950	Assert(cbLikely <= 15); /* See Intel spec. 2.3.11 "AVX Instruction Length" */
5951	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
5952	if (fSupportsNextRipSave)
5953	{
5954	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5955	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
5956	Assert(cbInstr == cbLikely);
5957	return cbInstr;
5958	}
5959	return cbLikely;
5960	}
5961	#endif
5962
5963
5964	/**
5965	* Advances the guest RIP by the number of bytes specified in @a cb. This does
5966	* not make use of any hardware features to determine the instruction length.
5967	*
5968	* @param pVCpu The cross context virtual CPU structure.
5969	* @param pCtx Pointer to the guest-CPU context.
5970	* @param cb RIP increment value in bytes.
5971	*/
5972	DECLINLINE(void) hmR0SvmAdvanceRipDumb(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
5973	{
5974	pCtx->rip += cb;
5975	HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
5976	}
5977	#undef HMSVM_UPDATE_INTR_SHADOW
5978
5979
5980	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5981	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
5982	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
5983
5984	/** @name \#VMEXIT handlers.
5985	* @{
5986	*/
5987
5988	/**
5989	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
5990	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
5991	*/
5992	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5993	{
5994	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
5995
5996	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
5997	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
5998	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
5999	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6000
6001	/*
6002	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to signal -before- the timer
6003	* fires if the current interrupt is our own timer or a some other host interrupt. We also cannot examine what
6004	* interrupt it is until the host actually take the interrupt.
6005	*
6006	* Going back to executing guest code here unconditionally causes random scheduling problems (observed on an
6007	* AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6008	*/
6009	return VINF_EM_RAW_INTERRUPT;
6010	}
6011
6012
6013	/**
6014	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6015	*/
6016	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6017	{
6018	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6019
6020	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6022	int rc = VINF_SUCCESS;
6023	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6024	return rc;
6025	}
6026
6027
6028	/**
6029	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6030	*/
6031	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6032	{
6033	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6034
6035	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6036	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6037	int rc = VINF_SUCCESS;
6038	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6039	return rc;
6040	}
6041
6042
6043	/**
6044	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6045	*/
6046	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6047	{
6048	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6049	PVM pVM = pVCpu->CTX_SUFF(pVM);
6050	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6051	if (RT_LIKELY(rc == VINF_SUCCESS))
6052	{
6053	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6054	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6055	}
6056	else
6057	{
6058	AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6059	rc = VERR_EM_INTERPRETER;
6060	}
6061	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6062	return rc;
6063	}
6064
6065
6066	/**
6067	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6068	*/
6069	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6070	{
6071	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6072	PVM pVM = pVCpu->CTX_SUFF(pVM);
6073	int rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6074	if (RT_LIKELY(rc == VINF_SUCCESS))
6075	{
6076	pSvmTransient->fUpdateTscOffsetting = true;
6077	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6078	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6079	}
6080	else
6081	{
6082	AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6083	rc = VERR_EM_INTERPRETER;
6084	}
6085	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6086	return rc;
6087	}
6088
6089
6090	/**
6091	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6092	*/
6093	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6094	{
6095	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6096	int rc = EMInterpretRdtscp(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
6097	if (RT_LIKELY(rc == VINF_SUCCESS))
6098	{
6099	pSvmTransient->fUpdateTscOffsetting = true;
6100	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6101	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6102	}
6103	else
6104	{
6105	AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtscp failed with %Rrc\n", rc));
6106	rc = VERR_EM_INTERPRETER;
6107	}
6108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
6109	return rc;
6110	}
6111
6112
6113	/**
6114	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6115	*/
6116	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6117	{
6118	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6119	int rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6120	if (RT_LIKELY(rc == VINF_SUCCESS))
6121	{
6122	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6123	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6124	}
6125	else
6126	{
6127	AssertMsgFailed(("hmR0SvmExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6128	rc = VERR_EM_INTERPRETER;
6129	}
6130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6131	return rc;
6132	}
6133
6134
6135	/**
6136	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
6137	*/
6138	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6139	{
6140	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6141	PVM pVM = pVCpu->CTX_SUFF(pVM);
6142	Assert(!pVM->hm.s.fNestedPaging);
6143	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6144
6145	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6146	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6147	if ( fSupportsDecodeAssists
6148	&& fSupportsNextRipSave)
6149	{
6150	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6151	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6152	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
6153	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
6154	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6155	return VBOXSTRICTRC_VAL(rcStrict);
6156	}
6157
6158	int rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, pCtx); /* Updates RIP if successful. */
6159	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
6160	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6161	return rc;
6162	}
6163
6164
6165	/**
6166	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6167	*/
6168	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6169	{
6170	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6171
6172	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 1);
6173	int rc = EMShouldContinueAfterHalt(pVCpu, pCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6174	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
6176	if (rc != VINF_SUCCESS)
6177	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6178	return rc;
6179	}
6180
6181
6182	/**
6183	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6184	*/
6185	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6186	{
6187	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6188	int rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6189	if (RT_LIKELY(rc == VINF_SUCCESS))
6190	{
6191	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6192	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6193	}
6194	else
6195	{
6196	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6197	rc = VERR_EM_INTERPRETER;
6198	}
6199	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6200	return rc;
6201	}
6202
6203
6204	/**
6205	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6206	*/
6207	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6208	{
6209	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6210	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6211	int rc = VBOXSTRICTRC_VAL(rc2);
6212	if ( rc == VINF_EM_HALT
6213	\|\| rc == VINF_SUCCESS)
6214	{
6215	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6216
6217	if ( rc == VINF_EM_HALT
6218	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
6219	{
6220	rc = VINF_SUCCESS;
6221	}
6222	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6223	}
6224	else
6225	{
6226	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
6227	rc = VERR_EM_INTERPRETER;
6228	}
6229	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
6230	("hmR0SvmExitMwait: EMInterpretMWait failed rc=%Rrc\n", rc));
6231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
6232	return rc;
6233	}
6234
6235
6236	/**
6237	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6238	* \#VMEXIT.
6239	*/
6240	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6241	{
6242	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6243	return VINF_EM_RESET;
6244	}
6245
6246
6247	/**
6248	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6249	*/
6250	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6251	{
6252	RT_NOREF(pCtx);
6253	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64\n", pSvmTransient->u64ExitCode));
6254	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6255	return VERR_SVM_UNEXPECTED_EXIT;
6256	}
6257
6258
6259	/**
6260	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6261	*/
6262	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6263	{
6264	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6265
6266	Log4(("hmR0SvmExitReadCRx: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0]);
6268
6269	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6270	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6271	if ( fSupportsDecodeAssists
6272	&& fSupportsNextRipSave)
6273	{
6274	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6275	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6276	if (fMovCRx)
6277	{
6278	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6279	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6280	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6281	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6282	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6283	return VBOXSTRICTRC_VAL(rcStrict);
6284	}
6285	/* else: SMSW instruction, fall back below to IEM for this. */
6286	}
6287
6288	VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6289	int rc = VBOXSTRICTRC_VAL(rc2);
6290	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3,
6291	("hmR0SvmExitReadCRx: EMInterpretInstruction failed rc=%Rrc\n", rc));
6292	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6293	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6294	return rc;
6295	}
6296
6297
6298	/**
6299	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6300	*/
6301	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6302	{
6303	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6304
6305	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0;
6306	Assert(iCrReg <= 15);
6307
6308	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6309	bool fDecodedInstr = false;
6310	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6311	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6312	if ( fSupportsDecodeAssists
6313	&& fSupportsNextRipSave)
6314	{
6315	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6316	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6317	if (fMovCRx)
6318	{
6319	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6320	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6321	Log4(("hmR0SvmExitWriteCRx: Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6322	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6323	fDecodedInstr = true;
6324	}
6325	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6326	}
6327
6328	if (!fDecodedInstr)
6329	{
6330	Log4(("hmR0SvmExitWriteCRx: iCrReg=%#x\n", iCrReg));
6331	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(pCtx), NULL);
6332	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6333	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6334	rcStrict = VERR_EM_INTERPRETER;
6335	}
6336
6337	if (rcStrict == VINF_SUCCESS)
6338	{
6339	switch (iCrReg)
6340	{
6341	case 0: /* CR0. */
6342	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6343	break;
6344
6345	case 3: /* CR3. */
6346	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
6347	break;
6348
6349	case 4: /* CR4. */
6350	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
6351	break;
6352
6353	case 8: /* CR8 (TPR). */
6354	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6355	break;
6356
6357	default:
6358	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6359	pSvmTransient->u64ExitCode, iCrReg));
6360	break;
6361	}
6362	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6363	}
6364	else
6365	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_CHANGE_MODE \|\| rcStrict == VINF_PGM_SYNC_CR3);
6366	return VBOXSTRICTRC_TODO(rcStrict);
6367	}
6368
6369
6370	/**
6371	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6372	* \#VMEXIT.
6373	*/
6374	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6375	{
6376	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6377	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6378	PVM pVM = pVCpu->CTX_SUFF(pVM);
6379
6380	int rc;
6381	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6382	{
6383	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6384	Log4(("MSR Write: idMsr=%#RX32\n", pCtx->ecx));
6385
6386	/* Handle TPR patching; intercepted LSTAR write. */
6387	if ( pVM->hm.s.fTPRPatchingActive
6388	&& pCtx->ecx == MSR_K8_LSTAR)
6389	{
6390	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6391	{
6392	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6393	int rc2 = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6394	AssertRC(rc2);
6395	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6396	}
6397	rc = VINF_SUCCESS;
6398	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6399	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6400	return rc;
6401	}
6402
6403	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6404	if (fSupportsNextRipSave)
6405	{
6406	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6407	if (RT_LIKELY(rc == VINF_SUCCESS))
6408	{
6409	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6410	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6411	}
6412	else
6413	AssertMsg( rc == VERR_EM_INTERPRETER
6414	\|\| rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: EMInterpretWrmsr failed rc=%Rrc\n", rc));
6415	}
6416	else
6417	{
6418	rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */));
6419	if (RT_LIKELY(rc == VINF_SUCCESS))
6420	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc); /* RIP updated by EMInterpretInstruction(). */
6421	else
6422	AssertMsg( rc == VERR_EM_INTERPRETER
6423	\|\| rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: WrMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6424	}
6425
6426	if (rc == VINF_SUCCESS)
6427	{
6428	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
6429	if ( pCtx->ecx >= MSR_IA32_X2APIC_START
6430	&& pCtx->ecx <= MSR_IA32_X2APIC_END)
6431	{
6432	/*
6433	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest(). When full APIC register
6434	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCB before
6435	* EMInterpretWrmsr() changes it.
6436	*/
6437	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6438	}
6439	else if (pCtx->ecx == MSR_K6_EFER)
6440	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
6441	else if (pCtx->ecx == MSR_IA32_TSC)
6442	pSvmTransient->fUpdateTscOffsetting = true;
6443	}
6444	}
6445	else
6446	{
6447	/* MSR Read access. */
6448	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6449	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ);
6450	Log4(("MSR Read: idMsr=%#RX32\n", pCtx->ecx));
6451
6452	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6453	if (fSupportsNextRipSave)
6454	{
6455	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6456	if (RT_LIKELY(rc == VINF_SUCCESS))
6457	{
6458	pCtx->rip = pVmcb->ctrl.u64NextRIP;
6459	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6460	}
6461	else
6462	AssertMsg( rc == VERR_EM_INTERPRETER
6463	\|\| rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: EMInterpretRdmsr failed rc=%Rrc\n", rc));
6464	}
6465	else
6466	{
6467	rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0));
6468	if (RT_UNLIKELY(rc != VINF_SUCCESS))
6469	{
6470	AssertMsg( rc == VERR_EM_INTERPRETER
6471	\|\| rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: RdMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6472	}
6473	/* RIP updated by EMInterpretInstruction(). */
6474	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6475	}
6476	}
6477
6478	/* RIP has been updated by EMInterpret[Rd\|Wr]msr() or EMInterpretInstruction(). */
6479	return rc;
6480	}
6481
6482
6483	/**
6484	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6485	*/
6486	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6487	{
6488	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6489	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6490
6491	/** @todo Stepping with nested-guest. */
6492	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6493	{
6494	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6495	if (pSvmTransient->fWasGuestDebugStateActive)
6496	{
6497	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6498	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6499	return VERR_SVM_UNEXPECTED_EXIT;
6500	}
6501
6502	/*
6503	* Lazy DR0-3 loading.
6504	*/
6505	if (!pSvmTransient->fWasHyperDebugStateActive)
6506	{
6507	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6508	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6509
6510	/* Don't intercept DRx read and writes. */
6511	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6512	pVmcb->ctrl.u16InterceptRdDRx = 0;
6513	pVmcb->ctrl.u16InterceptWrDRx = 0;
6514	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6515
6516	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6517	VMMRZCallRing3Disable(pVCpu);
6518	HM_DISABLE_PREEMPT();
6519
6520	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6521	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6522	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
6523
6524	HM_RESTORE_PREEMPT();
6525	VMMRZCallRing3Enable(pVCpu);
6526
6527	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6528	return VINF_SUCCESS;
6529	}
6530	}
6531
6532	/*
6533	* Interpret the read/writing of DRx.
6534	*/
6535	/** @todo Decode assist. */
6536	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6537	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6538	if (RT_LIKELY(rc == VINF_SUCCESS))
6539	{
6540	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6541	/** @todo CPUM should set this flag! */
6542	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6543	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6544	}
6545	else
6546	Assert(rc == VERR_EM_INTERPRETER);
6547	return VBOXSTRICTRC_TODO(rc);
6548	}
6549
6550
6551	/**
6552	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6553	*/
6554	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6555	{
6556	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6557	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6558	int rc = hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
6559	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6560	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6561	return rc;
6562	}
6563
6564
6565	/**
6566	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6567	*/
6568	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6569	{
6570	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6571
6572	/** @todo decode assists... */
6573	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6574	if (rcStrict == VINF_IEM_RAISED_XCPT)
6575	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6576
6577	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6578	Log4(("hmR0SvmExitXsetbv: New XCR0=%#RX64 fLoadSaveGuestXcr0=%d (cr4=%RX64) rcStrict=%Rrc\n",
6579	pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0, pCtx->cr4, VBOXSTRICTRC_VAL(rcStrict)));
6580
6581	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6582	return VBOXSTRICTRC_TODO(rcStrict);
6583	}
6584
6585
6586	/**
6587	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6588	*/
6589	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6590	{
6591	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6592
6593	/* I/O operation lookup arrays. */
6594	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
6595	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
6596	the result (in AL/AX/EAX). */
6597	Log4(("hmR0SvmExitIOInstr: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6598
6599	PVM pVM = pVCpu->CTX_SUFF(pVM);
6600	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6601
6602	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
6603	SVMIOIOEXITINFO IoExitInfo;
6604	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
6605	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
6606	uint32_t cbValue = s_aIOSize[uIOWidth];
6607	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
6608
6609	if (RT_UNLIKELY(!cbValue))
6610	{
6611	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
6612	return VERR_EM_INTERPRETER;
6613	}
6614
6615	VBOXSTRICTRC rcStrict;
6616	bool fUpdateRipAlready = false;
6617	if (IoExitInfo.n.u1STR)
6618	{
6619	#ifdef VBOX_WITH_2ND_IEM_STEP
6620	/* INS/OUTS - I/O String instruction. */
6621	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
6622	* in EXITINFO1? Investigate once this thing is up and running. */
6623	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
6624	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
6625	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
6626	static IEMMODE const s_aenmAddrMode[8] =
6627	{
6628	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
6629	};
6630	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
6631	if (enmAddrMode != (IEMMODE)-1)
6632	{
6633	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6634	if (cbInstr <= 15 && cbInstr >= 1)
6635	{
6636	Assert(cbInstr >= 1U + IoExitInfo.n.u1REP);
6637	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6638	{
6639	/* Don't know exactly how to detect whether u3SEG is valid, currently
6640	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
6641	2384 Opterons when only checking NRIP. */
6642	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6643	if ( fSupportsNextRipSave
6644	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
6645	{
6646	AssertMsg(IoExitInfo.n.u3SEG == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1REP,
6647	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3SEG, cbInstr, IoExitInfo.n.u1REP));
6648	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6649	IoExitInfo.n.u3SEG, true /fIoChecked/);
6650	}
6651	else if (cbInstr == 1U + IoExitInfo.n.u1REP)
6652	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6653	X86_SREG_DS, true /fIoChecked/);
6654	else
6655	rcStrict = IEMExecOne(pVCpu);
6656	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6657	}
6658	else
6659	{
6660	AssertMsg(IoExitInfo.n.u3SEG == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3SEG));
6661	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6662	true /fIoChecked/);
6663	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6664	}
6665	}
6666	else
6667	{
6668	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
6669	rcStrict = IEMExecOne(pVCpu);
6670	}
6671	}
6672	else
6673	{
6674	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
6675	rcStrict = IEMExecOne(pVCpu);
6676	}
6677	fUpdateRipAlready = true;
6678
6679	#else
6680	/* INS/OUTS - I/O String instruction. */
6681	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
6682
6683	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
6684	* in EXITINFO1? Investigate once this thing is up and running. */
6685
6686	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
6687	if (rcStrict == VINF_SUCCESS)
6688	{
6689	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6690	{
6691	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6692	(DISCPUMODE)pDis->uAddrMode, cbValue);
6693	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6694	}
6695	else
6696	{
6697	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6698	(DISCPUMODE)pDis->uAddrMode, cbValue);
6699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6700	}
6701	}
6702	else
6703	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
6704	#endif
6705	}
6706	else
6707	{
6708	/* IN/OUT - I/O instruction. */
6709	Assert(!IoExitInfo.n.u1REP);
6710
6711	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6712	{
6713	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
6714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
6715	}
6716	else
6717	{
6718	uint32_t u32Val = 0;
6719	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
6720	if (IOM_SUCCESS(rcStrict))
6721	{
6722	/* Save result of I/O IN instr. in AL/AX/EAX. */
6723	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
6724	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
6725	}
6726	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6727	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
6728
6729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
6730	}
6731	}
6732
6733	if (IOM_SUCCESS(rcStrict))
6734	{
6735	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
6736	if (!fUpdateRipAlready)
6737	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
6738
6739	/*
6740	* If any I/O breakpoints are armed, we need to check if one triggered
6741	* and take appropriate action.
6742	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
6743	*/
6744	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
6745	* execution engines about whether hyper BPs and such are pending. */
6746	uint32_t const uDr7 = pCtx->dr[7];
6747	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
6748	&& X86_DR7_ANY_RW_IO(uDr7)
6749	&& (pCtx->cr4 & X86_CR4_DE))
6750	\|\| DBGFBpIsHwIoArmed(pVM)))
6751	{
6752	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6753	VMMRZCallRing3Disable(pVCpu);
6754	HM_DISABLE_PREEMPT();
6755
6756	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
6757	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
6758
6759	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
6760	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
6761	{
6762	/* Raise #DB. */
6763	pVmcb->guest.u64DR6 = pCtx->dr[6];
6764	pVmcb->guest.u64DR7 = pCtx->dr[7];
6765	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
6766	hmR0SvmSetPendingXcptDB(pVCpu);
6767	}
6768	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
6769	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
6770	else if ( rcStrict2 != VINF_SUCCESS
6771	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
6772	rcStrict = rcStrict2;
6773	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
6774
6775	HM_RESTORE_PREEMPT();
6776	VMMRZCallRing3Enable(pVCpu);
6777	}
6778
6779	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6780	}
6781
6782	#ifdef VBOX_STRICT
6783	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6784	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
6785	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
6786	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
6787	else
6788	{
6789	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
6790	* statuses, that the VMM device and some others may return. See
6791	* IOM_SUCCESS() for guidance. */
6792	AssertMsg( RT_FAILURE(rcStrict)
6793	\|\| rcStrict == VINF_SUCCESS
6794	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
6795	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
6796	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
6797	\|\| rcStrict == VINF_EM_RAW_TO_R3
6798	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED
6799	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6800	}
6801	#endif
6802	return VBOXSTRICTRC_TODO(rcStrict);
6803	}
6804
6805
6806	/**
6807	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
6808	*/
6809	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6810	{
6811	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6812	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
6813
6814	PVM pVM = pVCpu->CTX_SUFF(pVM);
6815	Assert(pVM->hm.s.fNestedPaging);
6816
6817	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
6818
6819	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
6820	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6821	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
6822	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
6823
6824	Log4(("#NPF at CS:RIP=%04x:%#RX64 faultaddr=%RGp errcode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr, u32ErrCode));
6825
6826	#ifdef VBOX_HM_WITH_GUEST_PATCHING
6827	/* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions. */
6828	if ( pVM->hm.s.fTprPatchingAllowed
6829	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
6830	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
6831	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
6832	&& !CPUMIsGuestInLongModeEx(pCtx)
6833	&& !CPUMGetGuestCPL(pVCpu)
6834	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
6835	{
6836	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
6837	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6838
6839	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
6840	{
6841	/* Only attempt to patch the instruction once. */
6842	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
6843	if (!pPatch)
6844	return VINF_EM_HM_PATCH_TPR_INSTR;
6845	}
6846	}
6847	#endif
6848
6849	/*
6850	* Determine the nested paging mode.
6851	*/
6852	PGMMODE enmNestedPagingMode;
6853	#if HC_ARCH_BITS == 32
6854	if (CPUMIsGuestInLongModeEx(pCtx))
6855	enmNestedPagingMode = PGMMODE_AMD64_NX;
6856	else
6857	#endif
6858	enmNestedPagingMode = PGMGetHostMode(pVM);
6859
6860	/*
6861	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
6862	*/
6863	int rc;
6864	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
6865	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
6866	{
6867	/* If event delivery causes an MMIO #NPF, go back to instruction emulation as
6868	otherwise injecting the original pending event would most likely cause the same MMIO #NPF. */
6869	if (pVCpu->hm.s.Event.fPending)
6870	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6871
6872	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
6873	u32ErrCode);
6874	rc = VBOXSTRICTRC_VAL(rc2);
6875
6876	/*
6877	* If we succeed, resume guest execution.
6878	* If we fail in interpreting the instruction because we couldn't get the guest physical address
6879	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
6880	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
6881	* weird case. See @bugref{6043}.
6882	*/
6883	if ( rc == VINF_SUCCESS
6884	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
6885	\|\| rc == VERR_PAGE_NOT_PRESENT)
6886	{
6887	/* Successfully handled MMIO operation. */
6888	HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6889	rc = VINF_SUCCESS;
6890	}
6891	return rc;
6892	}
6893
6894	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
6895	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
6896	TRPMResetTrap(pVCpu);
6897
6898	Log4(("#NPF: PGMR0Trap0eHandlerNestedPaging returned %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
6899
6900	/*
6901	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
6902	*/
6903	if ( rc == VINF_SUCCESS
6904	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
6905	\|\| rc == VERR_PAGE_NOT_PRESENT)
6906	{
6907	/* We've successfully synced our shadow page tables. */
6908	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
6909	rc = VINF_SUCCESS;
6910	}
6911
6912	return rc;
6913	}
6914
6915
6916	/**
6917	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
6918	* \#VMEXIT.
6919	*/
6920	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6921	{
6922	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6923	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
6924
6925	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6926	hmR0SvmClearVirtIntrIntercept(pVmcb);
6927
6928	/* Deliver the pending interrupt/NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
6929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6930	return VINF_SUCCESS;
6931	}
6932
6933
6934	/**
6935	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
6936	* \#VMEXIT.
6937	*/
6938	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6939	{
6940	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6941
6942	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
6943
6944	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
6945	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
6946	#endif
6947
6948	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
6949	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
6950	{
6951	/*
6952	* AMD-V provides us with the exception which caused the TS; we collect
6953	* the information in the call to hmR0SvmCheckExitDueToEventDelivery.
6954	*/
6955	Log4(("hmR0SvmExitTaskSwitch: TS occurred during event delivery.\n"));
6956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
6957	return VINF_EM_RAW_INJECT_TRPM_EVENT;
6958	}
6959
6960	/** @todo Emulate task switch someday, currently just going back to ring-3 for
6961	* emulation. */
6962	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
6963	return VERR_EM_INTERPRETER;
6964	}
6965
6966
6967	/**
6968	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
6969	*/
6970	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6971	{
6972	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6973	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
6974
6975	bool fRipUpdated;
6976	VBOXSTRICTRC rcStrict = HMSvmVmmcall(pVCpu, pCtx, &fRipUpdated);
6977	if (RT_SUCCESS(rcStrict))
6978	{
6979	/* Only update the RIP if we're continuing guest execution and not
6980	in the case of say VINF_GIM_R3_HYPERCALL. */
6981	if ( rcStrict == VINF_SUCCESS
6982	&& !fRipUpdated)
6983	{
6984	hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3 /* cbInstr */);
6985	}
6986
6987	/* If the hypercall or TPR patching changes anything other than guest's general-purpose registers,
6988	we would need to reload the guest changed bits here before VM-entry. */
6989	return VBOXSTRICTRC_VAL(rcStrict);
6990	}
6991
6992	hmR0SvmSetPendingXcptUD(pVCpu);
6993	return VINF_SUCCESS;
6994	}
6995
6996
6997	/**
6998	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
6999	*/
7000	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7001	{
7002	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7004	return VINF_EM_RAW_INTERRUPT;
7005	}
7006
7007
7008	/**
7009	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7010	*/
7011	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7012	{
7013	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7014
7015	/* Clear NMI blocking. */
7016	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7017
7018	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7019	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7020	hmR0SvmClearIretIntercept(pVmcb);
7021
7022	/* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7023	return VINF_SUCCESS;
7024	}
7025
7026
7027	/**
7028	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_EXCEPTION_14).
7029	* Conditional \#VMEXIT.
7030	*/
7031	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7032	{
7033	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7034	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7035
7036	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7037
7038	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7039	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7040	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
7041	RTGCUINTPTR uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7042	PVM pVM = pVCpu->CTX_SUFF(pVM);
7043
7044	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
7045	if (pVM->hm.s.fNestedPaging)
7046	{
7047	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7048	if (!pSvmTransient->fVectoringDoublePF)
7049	{
7050	/* A genuine guest #PF, reflect it to the guest. */
7051	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7052	Log4(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RGv ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7053	uFaultAddress, u32ErrCode));
7054	}
7055	else
7056	{
7057	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7058	hmR0SvmSetPendingXcptDF(pVCpu);
7059	Log4(("Pending #DF due to vectoring #PF. NP\n"));
7060	}
7061	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7062	return VINF_SUCCESS;
7063	}
7064	#endif
7065
7066	Assert(!pVM->hm.s.fNestedPaging);
7067
7068	#ifdef VBOX_HM_WITH_GUEST_PATCHING
7069	/* Shortcut for APIC TPR reads and writes; only applicable to 32-bit guests. */
7070	if ( pVM->hm.s.fTprPatchingAllowed
7071	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7072	&& !(u32ErrCode & X86_TRAP_PF_P) /* Not present. */
7073	&& !CPUMIsGuestInLongModeEx(pCtx)
7074	&& !CPUMGetGuestCPL(pVCpu)
7075	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7076	{
7077	RTGCPHYS GCPhysApicBase;
7078	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7079	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7080
7081	/* Check if the page at the fault-address is the APIC base. */
7082	RTGCPHYS GCPhysPage;
7083	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7084	if ( rc2 == VINF_SUCCESS
7085	&& GCPhysPage == GCPhysApicBase)
7086	{
7087	/* Only attempt to patch the instruction once. */
7088	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7089	if (!pPatch)
7090	return VINF_EM_HM_PATCH_TPR_INSTR;
7091	}
7092	}
7093	#endif
7094
7095	Log4(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 u32ErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7096	pCtx->rip, u32ErrCode, pCtx->cr3));
7097
7098	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7099	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7100	if (pSvmTransient->fVectoringPF)
7101	{
7102	Assert(pVCpu->hm.s.Event.fPending);
7103	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7104	}
7105
7106	TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
7107	int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7108
7109	Log4(("#PF rc=%Rrc\n", rc));
7110
7111	if (rc == VINF_SUCCESS)
7112	{
7113	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7114	TRPMResetTrap(pVCpu);
7115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7116	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7117	return rc;
7118	}
7119	else if (rc == VINF_EM_RAW_GUEST_TRAP)
7120	{
7121	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7122
7123	if (!pSvmTransient->fVectoringDoublePF)
7124	{
7125	/* It's a guest page fault and needs to be reflected to the guest. */
7126	u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7127	TRPMResetTrap(pVCpu);
7128	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7129	}
7130	else
7131	{
7132	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7133	TRPMResetTrap(pVCpu);
7134	hmR0SvmSetPendingXcptDF(pVCpu);
7135	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7136	}
7137
7138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7139	return VINF_SUCCESS;
7140	}
7141
7142	TRPMResetTrap(pVCpu);
7143	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7144	return rc;
7145	}
7146
7147
7148	/**
7149	* \#VMEXIT handler for device-not-available exceptions (SVM_EXIT_EXCEPTION_7).
7150	* Conditional \#VMEXIT.
7151	*/
7152	HMSVM_EXIT_DECL hmR0SvmExitXcptNM(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7153	{
7154	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7155
7156	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7157	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7158	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7159
7160	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7161	VMMRZCallRing3Disable(pVCpu);
7162	HM_DISABLE_PREEMPT();
7163
7164	int rc;
7165	/* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
7166	if (pSvmTransient->fWasGuestFPUStateActive)
7167	{
7168	rc = VINF_EM_RAW_GUEST_TRAP;
7169	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
7170	}
7171	else
7172	{
7173	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
7174	Assert(!pSvmTransient->fWasGuestFPUStateActive);
7175	#endif
7176	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu); /* (No need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
7177	Assert( rc == VINF_EM_RAW_GUEST_TRAP
7178	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
7179	}
7180
7181	HM_RESTORE_PREEMPT();
7182	VMMRZCallRing3Enable(pVCpu);
7183
7184	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
7185	{
7186	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
7187	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
7189	pVCpu->hm.s.fPreloadGuestFpu = true;
7190	}
7191	else
7192	{
7193	/* Forward #NM to the guest. */
7194	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
7195	hmR0SvmSetPendingXcptNM(pVCpu);
7196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
7197	}
7198	return VINF_SUCCESS;
7199	}
7200
7201
7202	/**
7203	* \#VMEXIT handler for undefined opcode (SVM_EXIT_EXCEPTION_6).
7204	* Conditional \#VMEXIT.
7205	*/
7206	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7207	{
7208	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7209
7210	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7211	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7212	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7213
7214	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7215	if (pVCpu->hm.s.fGIMTrapXcptUD)
7216	{
7217	uint8_t cbInstr = 0;
7218	VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, pCtx, NULL /* pDis */, &cbInstr);
7219	if (rcStrict == VINF_SUCCESS)
7220	{
7221	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7222	hmR0SvmAdvanceRipDumb(pVCpu, pCtx, cbInstr);
7223	rc = VINF_SUCCESS;
7224	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7225	}
7226	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7227	rc = VINF_SUCCESS;
7228	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7229	rc = VINF_GIM_R3_HYPERCALL;
7230	else
7231	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7232	}
7233
7234	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7235	if (RT_FAILURE(rc))
7236	{
7237	hmR0SvmSetPendingXcptUD(pVCpu);
7238	rc = VINF_SUCCESS;
7239	}
7240
7241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7242	return rc;
7243	}
7244
7245
7246	/**
7247	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_EXCEPTION_16).
7248	* Conditional \#VMEXIT.
7249	*/
7250	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7251	{
7252	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7253
7254	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
7255	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7256	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7257
7258	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7259
7260	if (!(pCtx->cr0 & X86_CR0_NE))
7261	{
7262	PVM pVM = pVCpu->CTX_SUFF(pVM);
7263	PDISSTATE pDis = &pVCpu->hm.s.DisState;
7264	unsigned cbOp;
7265	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
7266	if (RT_SUCCESS(rc))
7267	{
7268	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7269	/** @todo FERR intercept when in nested-guest mode? */
7270	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
7271	if (RT_SUCCESS(rc))
7272	pCtx->rip += cbOp;
7273	}
7274	else
7275	Log4(("hmR0SvmExitXcptMF: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7276	return rc;
7277	}
7278
7279	hmR0SvmSetPendingXcptMF(pVCpu);
7280	return VINF_SUCCESS;
7281	}
7282
7283
7284	/**
7285	* \#VMEXIT handler for debug exceptions (SVM_EXIT_EXCEPTION_1). Conditional
7286	* \#VMEXIT.
7287	*/
7288	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7289	{
7290	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7291
7292	/* If this #DB is the result of delivering an event, go back to the interpreter. */
7293	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7294	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7295	{
7296	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7297	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7298	}
7299
7300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7301
7302	/* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data breakpoint). However, for both cases
7303	DR6 and DR7 are updated to what the exception handler expects. See AMD spec. 15.12.2 "#DB (Debug)". */
7304	PVM pVM = pVCpu->CTX_SUFF(pVM);
7305	PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7306	int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7307	if (rc == VINF_EM_RAW_GUEST_TRAP)
7308	{
7309	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7310	if (CPUMIsHyperDebugStateActive(pVCpu))
7311	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
7312
7313	/* Reflect the exception back to the guest. */
7314	hmR0SvmSetPendingXcptDB(pVCpu);
7315	rc = VINF_SUCCESS;
7316	}
7317
7318	/*
7319	* Update DR6.
7320	*/
7321	if (CPUMIsHyperDebugStateActive(pVCpu))
7322	{
7323	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7324	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7325	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7326	}
7327	else
7328	{
7329	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7330	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7331	}
7332
7333	return rc;
7334	}
7335
7336
7337	/**
7338	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_EXCEPTION_17).
7339	* Conditional \#VMEXIT.
7340	*/
7341	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7342	{
7343	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7344
7345	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7346	* shutdown VMEXIT. */
7347	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7348
7349	SVMEVENT Event;
7350	Event.u = 0;
7351	Event.n.u1Valid = 1;
7352	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7353	Event.n.u8Vector = X86_XCPT_AC;
7354	Event.n.u1ErrorCodeValid = 1;
7355	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7356	return VINF_SUCCESS;
7357	}
7358
7359
7360	/**
7361	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_EXCEPTION_3).
7362	* Conditional \#VMEXIT.
7363	*/
7364	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7365	{
7366	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7367
7368	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7369
7370	int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7371	if (rc == VINF_EM_RAW_GUEST_TRAP)
7372	{
7373	SVMEVENT Event;
7374	Event.u = 0;
7375	Event.n.u1Valid = 1;
7376	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7377	Event.n.u8Vector = X86_XCPT_BP;
7378	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7379	}
7380
7381	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
7382	return rc;
7383	}
7384
7385
7386	#ifdef VBOX_WITH_NESTED_HWVIRT
7387	/**
7388	* \#VMEXIT handler for #PF occuring while in nested-guest execution
7389	* (SVM_EXIT_EXCEPTION_14). Conditional \#VMEXIT.
7390	*/
7391	HMSVM_EXIT_DECL hmR0SvmExitXcptPFNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7392	{
7393	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7394
7395	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7396
7397	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7398	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7399	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
7400	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7401
7402	Log4(("#PFNested: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 u32ErrCode=%#RX32 CR3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7403	pCtx->rip, u32ErrCode, pCtx->cr3));
7404
7405	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7406	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7407	if (pSvmTransient->fVectoringPF)
7408	{
7409	Assert(pVCpu->hm.s.Event.fPending);
7410	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7411	}
7412
7413	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
7414
7415	TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
7416	int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7417
7418	Log4(("#PFNested: rc=%Rrc\n", rc));
7419
7420	if (rc == VINF_SUCCESS)
7421	{
7422	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7423	TRPMResetTrap(pVCpu);
7424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7425	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7426	return rc;
7427	}
7428
7429	if (rc == VINF_EM_RAW_GUEST_TRAP)
7430	{
7431	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7432
7433	if (!pSvmTransient->fVectoringDoublePF)
7434	{
7435	/* It's a nested-guest page fault and needs to be reflected to the nested-guest. */
7436	u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7437	TRPMResetTrap(pVCpu);
7438	hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7439	}
7440	else
7441	{
7442	/* A nested-guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7443	TRPMResetTrap(pVCpu);
7444	hmR0SvmSetPendingXcptDF(pVCpu);
7445	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7446	}
7447
7448	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7449	return VINF_SUCCESS;
7450	}
7451
7452	TRPMResetTrap(pVCpu);
7453	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7454	return rc;
7455	}
7456
7457
7458	/**
7459	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7460	*/
7461	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7462	{
7463	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7464
7465	/** @todo Stat. */
7466	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClgi); */
7467	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7468	VBOXSTRICTRC rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7469
7470	/*
7471	* The guest should no longer receive interrupts. Until VGIF is supported,
7472	* clear virtual interrupt intercepts here.
7473	*/
7474	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7475	hmR0SvmClearVirtIntrIntercept(pVmcb);
7476
7477	return VBOXSTRICTRC_VAL(rcStrict);
7478	}
7479
7480
7481	/**
7482	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7483	*/
7484	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7485	{
7486	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7487
7488	/** @todo Stat. */
7489	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitStgi); */
7490	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7491	VBOXSTRICTRC rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7492	return VBOXSTRICTRC_VAL(rcStrict);
7493	}
7494
7495
7496	/**
7497	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7498	*/
7499	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7500	{
7501	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7502
7503	/** @todo Stat. */
7504	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmload); */
7505	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7506	VBOXSTRICTRC rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7507	if (rcStrict == VINF_SUCCESS)
7508	{
7509	/* We skip flagging changes made to LSTAR, STAR, SFMASK and other MSRs as they are always re-loaded. */
7510	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7511	\| HM_CHANGED_GUEST_TR
7512	\| HM_CHANGED_GUEST_LDTR);
7513	}
7514	return VBOXSTRICTRC_VAL(rcStrict);
7515	}
7516
7517
7518	/**
7519	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7520	*/
7521	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7522	{
7523	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7524
7525	/** @todo Stat. */
7526	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmsave); */
7527	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7528	VBOXSTRICTRC rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7529	return VBOXSTRICTRC_VAL(rcStrict);
7530	}
7531
7532
7533	/**
7534	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
7535	*/
7536	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7537	{
7538	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7539	/** @todo Stat. */
7540	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpga); */
7541	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7542	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
7543	return VBOXSTRICTRC_VAL(rcStrict);
7544	}
7545
7546
7547	/**
7548	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
7549	*/
7550	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7551	{
7552	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7553	/** @todo Stat. */
7554	/* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmrun); */
7555	#if 0
7556	VBOXSTRICTRC rcStrict;
7557	uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7558	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
7559	Log4(("IEMExecDecodedVmrun: returned %d\n", VBOXSTRICTRC_VAL(rcStrict)));
7560	if (rcStrict == VINF_SUCCESS)
7561	{
7562	rcStrict = VINF_SVM_VMRUN;
7563	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7564	}
7565	return VBOXSTRICTRC_VAL(rcStrict);
7566	#endif
7567	return VERR_EM_INTERPRETER;
7568	}
7569
7570
7571	/**
7572	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7573	*/
7574	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7575	{
7576	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7577
7578	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7579	* shutdown VMEXIT. */
7580	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7581
7582	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7583	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_EXCEPTION_0;
7584	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7585	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7586	Assert(uVector <= X86_XCPT_LAST);
7587
7588	SVMEVENT Event;
7589	Event.u = 0;
7590	Event.n.u1Valid = 1;
7591	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7592	Event.n.u8Vector = uVector;
7593	switch (uVector)
7594	{
7595	case X86_XCPT_PF:
7596	case X86_XCPT_DF:
7597	case X86_XCPT_TS:
7598	case X86_XCPT_NP:
7599	case X86_XCPT_SS:
7600	case X86_XCPT_GP:
7601	case X86_XCPT_AC:
7602	{
7603	Event.n.u1ErrorCodeValid = 1;
7604	Event.n.u32ErrorCode = uErrCode;
7605	break;
7606	}
7607	}
7608
7609	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7610	return VINF_SUCCESS;
7611	}
7612
7613
7614	/**
7615	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_EXCEPTION_1).
7616	* Unconditional \#VMEXIT.
7617	*/
7618	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7619	{
7620	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7621
7622	/* If this #DB is the result of delivering an event, go back to the interpreter. */
7623	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7624	* shutdown VMEXIT. */
7625	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7626	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7627	{
7628	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7629	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7630	}
7631
7632	hmR0SvmSetPendingXcptDB(pVCpu);
7633	return VINF_SUCCESS;
7634	}
7635
7636
7637	/**
7638	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_EXCEPTION_3).
7639	* Conditional \#VMEXIT.
7640	*/
7641	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7642	{
7643	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7644
7645	/** @todo if triple-fault is returned in nested-guest scenario convert to a
7646	* shutdown VMEXIT. */
7647	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7648
7649	SVMEVENT Event;
7650	Event.u = 0;
7651	Event.n.u1Valid = 1;
7652	Event.n.u3Type = SVM_EVENT_EXCEPTION;
7653	Event.n.u8Vector = X86_XCPT_BP;
7654	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7655	return VINF_SUCCESS;
7656	}
7657
7658	#endif /* VBOX_WITH_NESTED_HWVIRT */
7659
7660
7661	/** @} */
7662

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 70262

Download in other formats: