HMSVMR0.cpp@ 93100

Last change on this file since 93100 was 93100, checked in by vboxsync, 3 years ago
VMM/HMSVMR0: doxygen fix
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 375.5 KB

Line
1	/* $Id: HMSVMR0.cpp 93100 2021-12-29 12:54:15Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/tm.h>
32	#include <VBox/vmm/em.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#include "HMInternal.h"
36	#include <VBox/vmm/vmcc.h>
37	#include <VBox/err.h>
38	#include "HMSVMR0.h"
39	#include "dtrace/VBoxVMM.h"
40
41	#ifdef DEBUG_ramshankar
42	# define HMSVM_SYNC_FULL_GUEST_STATE
43	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
44	# define HMSVM_ALWAYS_TRAP_PF
45	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
46	#endif
47
48
49	/*********************************************************************************************************************************
50	* Defined Constants And Macros *
51	*********************************************************************************************************************************/
52	#ifdef VBOX_WITH_STATISTICS
53	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
54	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
55	if ((u64ExitCode) == SVM_EXIT_NPF) \
56	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
57	else \
58	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
59	} while (0)
60
61	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
62	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
63	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
64	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
65	if ((u64ExitCode) == SVM_EXIT_NPF) \
66	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
67	else \
68	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
69	} while (0)
70	# endif
71	#else
72	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
73	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
74	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
75	# endif
76	#endif /* !VBOX_WITH_STATISTICS */
77
78	/** If we decide to use a function table approach this can be useful to
79	* switch to a "static DECLCALLBACK(int)". */
80	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
81
82	/**
83	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
84	* guest using hardware-assisted SVM.
85	*
86	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
87	* are swapped and restored across the world-switch and also registers like
88	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
89	* \#VMEXIT.
90	*/
91	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
92	\| CPUMCTX_EXTRN_RFLAGS \
93	\| CPUMCTX_EXTRN_RAX \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_CR0 \
97	\| CPUMCTX_EXTRN_CR2 \
98	\| CPUMCTX_EXTRN_CR3 \
99	\| CPUMCTX_EXTRN_TABLE_MASK \
100	\| CPUMCTX_EXTRN_DR6 \
101	\| CPUMCTX_EXTRN_DR7 \
102	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
103	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
104	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
105	\| CPUMCTX_EXTRN_HWVIRT \
106	\| CPUMCTX_EXTRN_INHIBIT_INT \
107	\| CPUMCTX_EXTRN_HM_SVM_MASK)
108
109	/**
110	* Subset of the guest-CPU state that is shared between the guest and host.
111	*/
112	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
113
114	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
115	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
116	do { \
117	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
118	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
119	} while (0)
120
121	/** Assert that the required state bits are fetched. */
122	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
123	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
124	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
125
126	/** Assert that preemption is disabled or covered by thread-context hooks. */
127	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
128	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
129
130	/** Assert that we haven't migrated CPUs when thread-context hooks are not
131	* used. */
132	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
133	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
134	("Illegal migration! Entered on CPU %u Current %u\n", \
135	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
136
137	/** Assert that we're not executing a nested-guest. */
138	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
139	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
140	#else
141	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
142	#endif
143
144	/** Assert that we're executing a nested-guest. */
145	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
146	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
147	#else
148	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
149	#endif
150
151	/** Macro for checking and returning from the using function for
152	* \#VMEXIT intercepts that maybe caused during delivering of another
153	* event in the guest. */
154	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
155	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
156	do \
157	{ \
158	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
159	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
160	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
161	else if ( rc == VINF_EM_RESET \
162	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
163	{ \
164	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
165	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
166	} \
167	else \
168	return rc; \
169	} while (0)
170	#else
171	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
172	do \
173	{ \
174	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
175	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
176	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
177	else \
178	return rc; \
179	} while (0)
180	#endif
181
182	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
183	* instruction that exited. */
184	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
185	do { \
186	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
187	(a_rc) = VINF_EM_DBG_STEPPED; \
188	} while (0)
189
190	/** Validate segment descriptor granularity bit. */
191	#ifdef VBOX_STRICT
192	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
193	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
194	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
195	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
196	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
197	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
198	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
199	#else
200	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
201	#endif
202
203	/**
204	* Exception bitmap mask for all contributory exceptions.
205	*
206	* Page fault is deliberately excluded here as it's conditional as to whether
207	* it's contributory or benign. Page faults are handled separately.
208	*/
209	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
210	\| RT_BIT(X86_XCPT_DE))
211
212	/**
213	* Mandatory/unconditional guest control intercepts.
214	*
215	* SMIs can and do happen in normal operation. We need not intercept them
216	* while executing the guest (or nested-guest).
217	*/
218	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
219	\| SVM_CTRL_INTERCEPT_NMI \
220	\| SVM_CTRL_INTERCEPT_INIT \
221	\| SVM_CTRL_INTERCEPT_RDPMC \
222	\| SVM_CTRL_INTERCEPT_CPUID \
223	\| SVM_CTRL_INTERCEPT_RSM \
224	\| SVM_CTRL_INTERCEPT_HLT \
225	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
226	\| SVM_CTRL_INTERCEPT_MSR_PROT \
227	\| SVM_CTRL_INTERCEPT_INVLPGA \
228	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
229	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
230	\| SVM_CTRL_INTERCEPT_VMRUN \
231	\| SVM_CTRL_INTERCEPT_SKINIT \
232	\| SVM_CTRL_INTERCEPT_WBINVD \
233	\| SVM_CTRL_INTERCEPT_MONITOR \
234	\| SVM_CTRL_INTERCEPT_MWAIT \
235	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
236	\| SVM_CTRL_INTERCEPT_XSETBV)
237
238	/** @name VMCB Clean Bits.
239	*
240	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
241	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
242	* memory.
243	*
244	* @{ */
245	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
246	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
247	/** I/O permission bitmap, MSR permission bitmap. */
248	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
249	/** ASID. */
250	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
251	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
252	V_INTR_VECTOR. */
253	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
254	/** Nested Paging: Nested CR3 (nCR3), PAT. */
255	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
256	/** Control registers (CR0, CR3, CR4, EFER). */
257	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
258	/** Debug registers (DR6, DR7). */
259	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
260	/** GDT, IDT limit and base. */
261	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
262	/** Segment register: CS, SS, DS, ES limit and base. */
263	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
264	/** CR2.*/
265	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
266	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
267	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
268	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
269	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
270	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
271	/** Mask of all valid VMCB Clean bits. */
272	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
273	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
274	\| HMSVM_VMCB_CLEAN_ASID \
275	\| HMSVM_VMCB_CLEAN_INT_CTRL \
276	\| HMSVM_VMCB_CLEAN_NP \
277	\| HMSVM_VMCB_CLEAN_CRX_EFER \
278	\| HMSVM_VMCB_CLEAN_DRX \
279	\| HMSVM_VMCB_CLEAN_DT \
280	\| HMSVM_VMCB_CLEAN_SEG \
281	\| HMSVM_VMCB_CLEAN_CR2 \
282	\| HMSVM_VMCB_CLEAN_LBR \
283	\| HMSVM_VMCB_CLEAN_AVIC)
284	/** @} */
285
286	/** @name SVM transient.
287	*
288	* A state structure for holding miscellaneous information across AMD-V
289	* VMRUN/\#VMEXIT operation, restored after the transition.
290	*
291	* @{ */
292	typedef struct SVMTRANSIENT
293	{
294	/** The host's rflags/eflags. */
295	RTCCUINTREG fEFlags;
296	/** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
297	uint64_t u64ExitCode;
298
299	/** The guest's TPR value used for TPR shadowing. */
300	uint8_t u8GuestTpr;
301	/** Alignment. */
302	uint8_t abAlignment0[7];
303
304	/** Pointer to the currently executing VMCB. */
305	PSVMVMCB pVmcb;
306
307	/** Whether we are currently executing a nested-guest. */
308	bool fIsNestedGuest;
309	/** Whether the guest debug state was active at the time of \#VMEXIT. */
310	bool fWasGuestDebugStateActive;
311	/** Whether the hyper debug state was active at the time of \#VMEXIT. */
312	bool fWasHyperDebugStateActive;
313	/** Whether the TSC offset mode needs to be updated. */
314	bool fUpdateTscOffsetting;
315	/** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
316	bool fRestoreTscAuxMsr;
317	/** Whether the \#VMEXIT was caused by a page-fault during delivery of a
318	* contributary exception or a page-fault. */
319	bool fVectoringDoublePF;
320	/** Whether the \#VMEXIT was caused by a page-fault during delivery of an
321	* external interrupt or NMI. */
322	bool fVectoringPF;
323	/** Padding. */
324	bool afPadding0;
325	} SVMTRANSIENT;
326	/** Pointer to SVM transient state. */
327	typedef SVMTRANSIENT *PSVMTRANSIENT;
328	/** Pointer to a const SVM transient state. */
329	typedef const SVMTRANSIENT *PCSVMTRANSIENT;
330
331	AssertCompileSizeAlignment(SVMTRANSIENT, sizeof(uint64_t));
332	AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
333	AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
334	/** @} */
335
336	/**
337	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
338	*/
339	typedef enum SVMMSREXITREAD
340	{
341	/** Reading this MSR causes a \#VMEXIT. */
342	SVMMSREXIT_INTERCEPT_READ = 0xb,
343	/** Reading this MSR does not cause a \#VMEXIT. */
344	SVMMSREXIT_PASSTHRU_READ
345	} SVMMSREXITREAD;
346
347	/**
348	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
349	*/
350	typedef enum SVMMSREXITWRITE
351	{
352	/** Writing to this MSR causes a \#VMEXIT. */
353	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
354	/** Writing to this MSR does not cause a \#VMEXIT. */
355	SVMMSREXIT_PASSTHRU_WRITE
356	} SVMMSREXITWRITE;
357
358	/**
359	* SVM \#VMEXIT handler.
360	*
361	* @returns Strict VBox status code.
362	* @param pVCpu The cross context virtual CPU structure.
363	* @param pSvmTransient Pointer to the SVM-transient structure.
364	*/
365	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
366
367
368	/*********************************************************************************************************************************
369	* Internal Functions *
370	*********************************************************************************************************************************/
371	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
372	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
373
374
375	/** @name \#VMEXIT handlers.
376	* @{
377	*/
378	static FNSVMEXITHANDLER hmR0SvmExitIntr;
379	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
380	static FNSVMEXITHANDLER hmR0SvmExitInvd;
381	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
382	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
383	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
384	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
385	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
386	static FNSVMEXITHANDLER hmR0SvmExitHlt;
387	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
388	static FNSVMEXITHANDLER hmR0SvmExitMwait;
389	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
390	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
391	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
392	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
393	static FNSVMEXITHANDLER hmR0SvmExitMsr;
394	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
395	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
396	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
397	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
398	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
399	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
400	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
401	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
402	static FNSVMEXITHANDLER hmR0SvmExitPause;
403	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
404	static FNSVMEXITHANDLER hmR0SvmExitIret;
405	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
406	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
407	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
408	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
409	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
410	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
411	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
412	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
413	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
414	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
415	static FNSVMEXITHANDLER hmR0SvmExitClgi;
416	static FNSVMEXITHANDLER hmR0SvmExitStgi;
417	static FNSVMEXITHANDLER hmR0SvmExitVmload;
418	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
419	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
420	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
421	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
422	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
423	#endif
424	/** @} */
425
426	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
427	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
428	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
429	#endif
430	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
431
432
433	/*********************************************************************************************************************************
434	* Global Variables *
435	*********************************************************************************************************************************/
436	/** Ring-0 memory object for the IO bitmap. */
437	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
438	/** Physical address of the IO bitmap. */
439	static RTHCPHYS g_HCPhysIOBitmap;
440	/** Pointer to the IO bitmap. */
441	static R0PTRTYPE(void *) g_pvIOBitmap;
442
443	#ifdef VBOX_STRICT
444	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
445	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
446	# define HMSVM_LOG_CS RT_BIT_32(2)
447	# define HMSVM_LOG_SS RT_BIT_32(3)
448	# define HMSVM_LOG_FS RT_BIT_32(4)
449	# define HMSVM_LOG_GS RT_BIT_32(5)
450	# define HMSVM_LOG_LBR RT_BIT_32(6)
451	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
452	\| HMSVM_LOG_CR_REGS \
453	\| HMSVM_LOG_CS \
454	\| HMSVM_LOG_SS \
455	\| HMSVM_LOG_FS \
456	\| HMSVM_LOG_GS \
457	\| HMSVM_LOG_LBR)
458
459	/**
460	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
461	*
462	* @param pVCpu The cross context virtual CPU structure.
463	* @param pVmcb Pointer to the VM control block.
464	* @param pszPrefix Log prefix.
465	* @param fFlags Log flags, see HMSVM_LOG_XXX.
466	* @param uVerbose The verbosity level, currently unused.
467	*/
468	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
469	{
470	RT_NOREF2(pVCpu, uVerbose);
471	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
472
473	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
474	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
475
476	if (fFlags & HMSVM_LOG_RBP_RSP)
477	{
478	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
479	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
480	}
481
482	if (fFlags & HMSVM_LOG_CR_REGS)
483	{
484	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
485	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
486	}
487
488	if (fFlags & HMSVM_LOG_CS)
489	{
490	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
491	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
492	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
493	}
494	if (fFlags & HMSVM_LOG_SS)
495	{
496	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
497	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
498	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
499	}
500	if (fFlags & HMSVM_LOG_FS)
501	{
502	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
503	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
504	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
505	}
506	if (fFlags & HMSVM_LOG_GS)
507	{
508	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
509	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
510	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
511	}
512
513	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
514	if (fFlags & HMSVM_LOG_LBR)
515	{
516	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
517	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
518	}
519	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
520	}
521	#endif /* VBOX_STRICT */
522
523
524	/**
525	* Sets up and activates AMD-V on the current CPU.
526	*
527	* @returns VBox status code.
528	* @param pHostCpu The HM physical-CPU structure.
529	* @param pVM The cross context VM structure. Can be
530	* NULL after a resume!
531	* @param pvCpuPage Pointer to the global CPU page.
532	* @param HCPhysCpuPage Physical address of the global CPU page.
533	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
534	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
535	* unused).
536	*/
537	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
538	PCSUPHWVIRTMSRS pHwvirtMsrs)
539	{
540	Assert(!fEnabledByHost);
541	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
542	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
543	Assert(pvCpuPage); NOREF(pvCpuPage);
544	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
545
546	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
547
548	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
549	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
550
551	/*
552	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
553	*/
554	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
555	if (u64HostEfer & MSR_K6_EFER_SVME)
556	{
557	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
558	if ( pVM
559	&& pVM->hm.s.svm.fIgnoreInUseError)
560	pHostCpu->fIgnoreAMDVInUseError = true;
561
562	if (!pHostCpu->fIgnoreAMDVInUseError)
563	{
564	ASMSetFlags(fEFlags);
565	return VERR_SVM_IN_USE;
566	}
567	}
568
569	/* Turn on AMD-V in the EFER MSR. */
570	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
571
572	/* Write the physical page address where the CPU will store the host state while executing the VM. */
573	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
574
575	/* Restore interrupts. */
576	ASMSetFlags(fEFlags);
577
578	/*
579	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
580	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
581	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
582	* entirely with before executing any guest code.
583	*/
584	pHostCpu->fFlushAsidBeforeUse = true;
585
586	/*
587	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
588	*/
589	++pHostCpu->cTlbFlushes;
590
591	return VINF_SUCCESS;
592	}
593
594
595	/**
596	* Deactivates AMD-V on the current CPU.
597	*
598	* @returns VBox status code.
599	* @param pHostCpu The HM physical-CPU structure.
600	* @param pvCpuPage Pointer to the global CPU page.
601	* @param HCPhysCpuPage Physical address of the global CPU page.
602	*/
603	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
604	{
605	RT_NOREF1(pHostCpu);
606	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
607	AssertReturn( HCPhysCpuPage
608	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
609	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
610
611	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
612	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
613
614	/* Turn off AMD-V in the EFER MSR. */
615	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
616	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
617
618	/* Invalidate host state physical address. */
619	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
620
621	/* Restore interrupts. */
622	ASMSetFlags(fEFlags);
623
624	return VINF_SUCCESS;
625	}
626
627
628	/**
629	* Does global AMD-V initialization (called during module initialization).
630	*
631	* @returns VBox status code.
632	*/
633	VMMR0DECL(int) SVMR0GlobalInit(void)
634	{
635	/*
636	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
637	* intercept all IO accesses, it's done once globally here instead of per-VM.
638	*/
639	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
640	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
641	if (RT_FAILURE(rc))
642	return rc;
643
644	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
645	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
646
647	/* Set all bits to intercept all IO accesses. */
648	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
649
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Does global AMD-V termination (called during module termination).
656	*/
657	VMMR0DECL(void) SVMR0GlobalTerm(void)
658	{
659	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
660	{
661	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
662	g_pvIOBitmap = NULL;
663	g_HCPhysIOBitmap = 0;
664	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
665	}
666	}
667
668
669	/**
670	* Frees any allocated per-VCPU structures for a VM.
671	*
672	* @param pVM The cross context VM structure.
673	*/
674	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
675	{
676	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
677	{
678	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
679	AssertPtr(pVCpu);
680
681	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
682	{
683	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
684	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
685	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
686	}
687
688	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
689	{
690	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
691	pVCpu->hmr0.s.svm.pVmcb = NULL;
692	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
693	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
694	}
695
696	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
697	{
698	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
699	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
700	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
701	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
702	}
703	}
704	}
705
706
707	/**
708	* Sets pfnVMRun to the best suited variant.
709	*
710	* This must be called whenever anything changes relative to the SVMR0VMRun
711	* variant selection:
712	* - pVCpu->hm.s.fLoadSaveGuestXcr0
713	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
714	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
715	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
716	* - Perhaps: CPUMCTX.fXStateMask (windows only)
717	*
718	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
719	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
720	*/
721	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
722	{
723	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
724	{
725	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
726	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
727	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
728	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
729	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
730	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
731	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
732	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
733	};
734	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
735	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
736	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
737	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
738	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
739	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
740	}
741
742
743	/**
744	* Selector FNHMSVMVMRUN implementation.
745	*/
746	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
747	{
748	hmR0SvmUpdateVmRunFunction(pVCpu);
749	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
750	}
751
752
753	/**
754	* Does per-VM AMD-V initialization.
755	*
756	* @returns VBox status code.
757	* @param pVM The cross context VM structure.
758	*/
759	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
760	{
761	int rc = VERR_INTERNAL_ERROR_5;
762
763	/*
764	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
765	*/
766	uint32_t u32Family;
767	uint32_t u32Model;
768	uint32_t u32Stepping;
769	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
770	{
771	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
772	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
773	}
774
775	/*
776	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
777	*/
778	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
779	{
780	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
781	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
782	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
783	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
784	}
785
786	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
787	{
788	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
789
790	/*
791	* Initialize the hardware-assisted SVM guest-execution handler.
792	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
793	*/
794	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
795
796	/*
797	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
798	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
799	*/
800	/** @todo Does this need to be below 4G? */
801	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
802	if (RT_FAILURE(rc))
803	goto failure_cleanup;
804
805	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
806	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
807	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
808	ASMMemZeroPage(pvVmcbHost);
809
810	/*
811	* Allocate one page for the guest-state VMCB.
812	*/
813	/** @todo Does this need to be below 4G? */
814	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
815	if (RT_FAILURE(rc))
816	goto failure_cleanup;
817
818	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
819	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
820	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
821	ASMMemZeroPage(pVCpu->hmr0.s.svm.pVmcb);
822
823	/*
824	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
825	* SVM to not require one.
826	*/
827	/** @todo Does this need to be below 4G? */
828	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
829	false /* fExecutable */);
830	if (RT_FAILURE(rc))
831	goto failure_cleanup;
832
833	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
834	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
835	/* Set all bits to intercept all MSR accesses (changed later on). */
836	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
837	}
838
839	return VINF_SUCCESS;
840
841	failure_cleanup:
842	hmR0SvmFreeStructs(pVM);
843	return rc;
844	}
845
846
847	/**
848	* Does per-VM AMD-V termination.
849	*
850	* @returns VBox status code.
851	* @param pVM The cross context VM structure.
852	*/
853	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
854	{
855	hmR0SvmFreeStructs(pVM);
856	return VINF_SUCCESS;
857	}
858
859
860	/**
861	* Returns whether the VMCB Clean Bits feature is supported.
862	*
863	* @returns @c true if supported, @c false otherwise.
864	* @param pVCpu The cross context virtual CPU structure.
865	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
866	*/
867	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
868	{
869	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
870	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
871	if (!fIsNestedGuest)
872	return fHostVmcbCleanBits;
873	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
874	}
875
876
877	/**
878	* Returns whether the decode assists feature is supported.
879	*
880	* @returns @c true if supported, @c false otherwise.
881	* @param pVCpu The cross context virtual CPU structure.
882	*/
883	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
884	{
885	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
886	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
887	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
888	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
889	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
890	#endif
891	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
892	}
893
894
895	/**
896	* Returns whether the NRIP_SAVE feature is supported.
897	*
898	* @returns @c true if supported, @c false otherwise.
899	* @param pVCpu The cross context virtual CPU structure.
900	*/
901	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
902	{
903	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
904	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
905	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
906	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
907	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
908	#endif
909	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
910	}
911
912
913	/**
914	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
915	*
916	* @param pVCpu The cross context virtual CPU structure.
917	* @param pbMsrBitmap Pointer to the MSR bitmap.
918	* @param idMsr The MSR for which the permissions are being set.
919	* @param enmRead MSR read permissions.
920	* @param enmWrite MSR write permissions.
921	*
922	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
923	* caller needs to take care of this.
924	*/
925	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
926	SVMMSREXITWRITE enmWrite)
927	{
928	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
929	uint16_t offMsrpm;
930	uint8_t uMsrpmBit;
931	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
932	AssertRC(rc);
933
934	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
935	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
936
937	pbMsrBitmap += offMsrpm;
938	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
939	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
940	else
941	{
942	if (!fInNestedGuestMode)
943	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
944	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
945	else
946	{
947	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
948	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
949	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
950	else
951	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
952	}
953	#endif
954	}
955
956	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
957	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
958	else
959	{
960	if (!fInNestedGuestMode)
961	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
962	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
963	else
964	{
965	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
966	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
967	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
968	else
969	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
970	}
971	#endif
972	}
973	}
974
975
976	/**
977	* Sets up AMD-V for the specified VM.
978	* This function is only called once per-VM during initalization.
979	*
980	* @returns VBox status code.
981	* @param pVM The cross context VM structure.
982	*/
983	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
984	{
985	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
986	AssertReturn(pVM, VERR_INVALID_PARAMETER);
987
988	/*
989	* Validate and copy over some parameters.
990	*/
991	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
992	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
993	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
994	pVM->hmr0.s.fNestedPaging = fNestedPaging;
995	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
996
997	/*
998	* Determin some configuration parameters.
999	*/
1000	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
1001	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
1002	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
1003
1004	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
1005	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
1006
1007	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1008	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
1009	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
1010
1011	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
1012	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
1013	#endif
1014
1015	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
1016	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
1017	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
1018	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
1019
1020	/* Always trap #AC for reasons of security. */
1021	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
1022
1023	/* Always trap #DB for reasons of security. */
1024	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
1025
1026	/* Trap exceptions unconditionally (debug purposes). */
1027	#ifdef HMSVM_ALWAYS_TRAP_PF
1028	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
1029	#endif
1030	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1031	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1032	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1033	\| RT_BIT_32(X86_XCPT_DE)
1034	\| RT_BIT_32(X86_XCPT_NM)
1035	\| RT_BIT_32(X86_XCPT_UD)
1036	\| RT_BIT_32(X86_XCPT_NP)
1037	\| RT_BIT_32(X86_XCPT_SS)
1038	\| RT_BIT_32(X86_XCPT_GP)
1039	\| RT_BIT_32(X86_XCPT_PF)
1040	\| RT_BIT_32(X86_XCPT_MF)
1041	;
1042	#endif
1043
1044	/* Apply the exceptions intercepts needed by the GIM provider. */
1045	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1046	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1047
1048	/* The mesa 3d driver hack needs #GP. */
1049	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1050	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1051
1052	/* Set up unconditional intercepts and conditions. */
1053	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1054	\| SVM_CTRL_INTERCEPT_VMMCALL
1055	\| SVM_CTRL_INTERCEPT_VMSAVE
1056	\| SVM_CTRL_INTERCEPT_VMLOAD
1057	\| SVM_CTRL_INTERCEPT_CLGI
1058	\| SVM_CTRL_INTERCEPT_STGI;
1059
1060	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1061	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1062	#endif
1063
1064	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1065	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1066	{
1067	/* Virtualized VMSAVE/VMLOAD. */
1068	if (fUseVirtVmsaveVmload)
1069	{
1070	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1071	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1072	\| SVM_CTRL_INTERCEPT_VMLOAD);
1073	}
1074	else
1075	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1076
1077	/* Virtual GIF. */
1078	if (fUseVGif)
1079	{
1080	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1081	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1082	\| SVM_CTRL_INTERCEPT_STGI);
1083	}
1084	else
1085	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1086	}
1087	else
1088	#endif
1089	{
1090	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1091	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1092	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1093	}
1094
1095	/* CR4 writes must always be intercepted for tracking PGM mode changes. */
1096	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1097
1098	/* Intercept all DRx reads and writes by default. Changed later on. */
1099	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1100	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1101
1102	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1103	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1104
1105	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1106	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1107	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1108
1109	/* Set the IO permission bitmap physical addresses. */
1110	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1111
1112	/* LBR virtualization. */
1113	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1114
1115	/* The host ASID MBZ, for the guest start with 1. */
1116	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1117
1118	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1119	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1120
1121	/* Without Nested Paging, we need additionally intercepts. */
1122	if (!fNestedPaging)
1123	{
1124	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1125	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1126	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1127
1128	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1129	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1130	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1131
1132	/* Page faults must be intercepted to implement shadow paging. */
1133	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1134	}
1135
1136	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1137	if (fUsePauseFilter)
1138	{
1139	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1140	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1141	if (fPauseFilterThreshold)
1142	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1143	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1144	}
1145
1146	/*
1147	* Setup the MSR permission bitmap.
1148	* The following MSRs are saved/restored automatically during the world-switch.
1149	* Don't intercept guest read/write accesses to these MSRs.
1150	*/
1151	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1152	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1153	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1154	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1155	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1156	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1157	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1158	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1159	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1160	{
1161	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1162	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1163	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1164	}
1165	else
1166	{
1167	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1168	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1169	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1170	}
1171	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1172
1173	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1174	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1175
1176	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1177	{
1178	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1179	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1180	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1181	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1182
1183	/* Copy the VMCB control area. */
1184	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1185
1186	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1187	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1188	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1189	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1190
1191	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1192	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1193
1194	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1195	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1196	}
1197
1198	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1199	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1200	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1201	#else
1202	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1203	#endif
1204	return VINF_SUCCESS;
1205	}
1206
1207
1208	/**
1209	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1210	*
1211	* @returns Pointer to the current context VMCB.
1212	* @param pVCpu The cross context virtual CPU structure.
1213	*/
1214	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1215	{
1216	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1217	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1218	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1219	#endif
1220	return pVCpu->hmr0.s.svm.pVmcb;
1221	}
1222
1223
1224	/**
1225	* Gets a pointer to the nested-guest VMCB cache.
1226	*
1227	* @returns Pointer to the nested-guest VMCB cache.
1228	* @param pVCpu The cross context virtual CPU structure.
1229	*/
1230	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1231	{
1232	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1233	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1234	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1235	#else
1236	RT_NOREF(pVCpu);
1237	return NULL;
1238	#endif
1239	}
1240
1241
1242	/**
1243	* Invalidates a guest page by guest virtual address.
1244	*
1245	* @returns VBox status code.
1246	* @param pVCpu The cross context virtual CPU structure.
1247	* @param GCVirt Guest virtual address of the page to invalidate.
1248	*/
1249	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1250	{
1251	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1252
1253	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1254
1255	/* Skip it if a TLB flush is already pending. */
1256	if (!fFlushPending)
1257	{
1258	Log4Func(("%#RGv\n", GCVirt));
1259
1260	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1261	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1262
1263	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1264	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1265	}
1266	return VINF_SUCCESS;
1267	}
1268
1269
1270	/**
1271	* Flushes the appropriate tagged-TLB entries.
1272	*
1273	* @param pHostCpu The HM physical-CPU structure.
1274	* @param pVCpu The cross context virtual CPU structure.
1275	* @param pVmcb Pointer to the VM control block.
1276	*/
1277	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1278	{
1279	/*
1280	* Force a TLB flush for the first world switch if the current CPU differs from the one
1281	* we ran on last. This can happen both for start & resume due to long jumps back to
1282	* ring-3.
1283	*
1284	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1285	* correlation between it and the physical CPU.
1286	*
1287	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1288	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1289	*/
1290	bool fNewAsid = false;
1291	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1292	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1293	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1294	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1295	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1296	#endif
1297	)
1298	{
1299	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1300	pVCpu->hmr0.s.fForceTLBFlush = true;
1301	fNewAsid = true;
1302	}
1303
1304	/* Set TLB flush state as checked until we return from the world switch. */
1305	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1306
1307	/* Check for explicit TLB flushes. */
1308	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1309	{
1310	pVCpu->hmr0.s.fForceTLBFlush = true;
1311	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1312	}
1313
1314	/*
1315	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1316	* This Host CPU requirement takes precedence.
1317	*/
1318	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1319	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1320	{
1321	pHostCpu->uCurrentAsid = 1;
1322	pVCpu->hmr0.s.uCurrentAsid = 1;
1323	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1324	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1325	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1326
1327	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1328	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1329	}
1330	else
1331	{
1332	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1333	if (pVCpu->hmr0.s.fForceTLBFlush)
1334	{
1335	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1336	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1337
1338	if (fNewAsid)
1339	{
1340	++pHostCpu->uCurrentAsid;
1341
1342	bool fHitASIDLimit = false;
1343	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1344	{
1345	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1346	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1347	fHitASIDLimit = true;
1348	}
1349
1350	if ( fHitASIDLimit
1351	\|\| pHostCpu->fFlushAsidBeforeUse)
1352	{
1353	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1354	pHostCpu->fFlushAsidBeforeUse = false;
1355	}
1356
1357	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1358	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1359	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1360	}
1361	else
1362	{
1363	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1364	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1365	else
1366	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1367	}
1368
1369	pVCpu->hmr0.s.fForceTLBFlush = false;
1370	}
1371	}
1372
1373	/* Update VMCB with the ASID. */
1374	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1375	{
1376	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1377	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1378	}
1379
1380	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1381	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1382	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1383	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1384	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1385	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1386	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1387	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1388
1389	#ifdef VBOX_WITH_STATISTICS
1390	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1392	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1393	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1394	{
1395	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1396	}
1397	else
1398	{
1399	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1400	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1401	}
1402	#endif
1403	}
1404
1405
1406	/**
1407	* Sets an exception intercept in the specified VMCB.
1408	*
1409	* @param pVmcb Pointer to the VM control block.
1410	* @param uXcpt The exception (X86_XCPT_*).
1411	*/
1412	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1413	{
1414	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1415	{
1416	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1417	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1418	}
1419	}
1420
1421
1422	/**
1423	* Clears an exception intercept in the specified VMCB.
1424	*
1425	* @param pVCpu The cross context virtual CPU structure.
1426	* @param pVmcb Pointer to the VM control block.
1427	* @param uXcpt The exception (X86_XCPT_*).
1428	*
1429	* @remarks This takes into account if we're executing a nested-guest and only
1430	* removes the exception intercept if both the guest -and- nested-guest
1431	* are not intercepting it.
1432	*/
1433	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1434	{
1435	Assert(uXcpt != X86_XCPT_DB);
1436	Assert(uXcpt != X86_XCPT_AC);
1437	Assert(uXcpt != X86_XCPT_GP);
1438	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1439	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1440	{
1441	bool fRemove = true;
1442	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1443	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1444	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1445	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1446	{
1447	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1448	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1449	}
1450	# else
1451	RT_NOREF(pVCpu);
1452	# endif
1453	if (fRemove)
1454	{
1455	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1456	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1457	}
1458	}
1459	#else
1460	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1461	#endif
1462	}
1463
1464
1465	/**
1466	* Sets a control intercept in the specified VMCB.
1467	*
1468	* @param pVmcb Pointer to the VM control block.
1469	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1470	*/
1471	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1472	{
1473	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1474	{
1475	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1476	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1477	}
1478	}
1479
1480
1481	/**
1482	* Clears a control intercept in the specified VMCB.
1483	*
1484	* @returns @c true if the intercept is still set, @c false otherwise.
1485	* @param pVCpu The cross context virtual CPU structure.
1486	* @param pVmcb Pointer to the VM control block.
1487	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1488	*
1489	* @remarks This takes into account if we're executing a nested-guest and only
1490	* removes the control intercept if both the guest -and- nested-guest
1491	* are not intercepting it.
1492	*/
1493	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1494	{
1495	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1496	{
1497	bool fRemove = true;
1498	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1499	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1500	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1501	{
1502	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1503	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1504	}
1505	#else
1506	RT_NOREF(pVCpu);
1507	#endif
1508	if (fRemove)
1509	{
1510	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1511	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1512	}
1513	}
1514
1515	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1516	}
1517
1518
1519	/**
1520	* Exports the guest (or nested-guest) CR0 into the VMCB.
1521	*
1522	* @param pVCpu The cross context virtual CPU structure.
1523	* @param pVmcb Pointer to the VM control block.
1524	*
1525	* @remarks This assumes we always pre-load the guest FPU.
1526	* @remarks No-long-jump zone!!!
1527	*/
1528	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1529	{
1530	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1531
1532	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1533	uint64_t const uGuestCr0 = pCtx->cr0;
1534	uint64_t uShadowCr0 = uGuestCr0;
1535
1536	/* Always enable caching. */
1537	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1538
1539	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1540	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1541	{
1542	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1543	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1544	}
1545
1546	/*
1547	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1548	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1549	* see @bugref{7243#c103}.
1550	*/
1551	if (!(uGuestCr0 & X86_CR0_NE))
1552	{
1553	uShadowCr0 \|= X86_CR0_NE;
1554	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1555	}
1556	else
1557	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1558
1559	/*
1560	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1561	*
1562	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1563	* see @bugref{6944}.
1564	*
1565	* We also don't ever want to honor weird things like cache disable from the guest.
1566	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1567	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1568	*/
1569	if (uShadowCr0 == uGuestCr0)
1570	{
1571	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1572	{
1573	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1574	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1575	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1576	}
1577	else
1578	{
1579	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1580	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1581	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1582	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1583	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1584	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1585	}
1586	}
1587	else
1588	{
1589	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1590	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1591	}
1592	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1593
1594	Assert(!RT_HI_U32(uShadowCr0));
1595	if (pVmcb->guest.u64CR0 != uShadowCr0)
1596	{
1597	pVmcb->guest.u64CR0 = uShadowCr0;
1598	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1599	}
1600	}
1601
1602
1603	/**
1604	* Exports the guest (or nested-guest) CR3 into the VMCB.
1605	*
1606	* @param pVCpu The cross context virtual CPU structure.
1607	* @param pVmcb Pointer to the VM control block.
1608	*
1609	* @remarks No-long-jump zone!!!
1610	*/
1611	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1612	{
1613	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1614
1615	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1616	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1617	if (pVM->hmr0.s.fNestedPaging)
1618	{
1619	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1620	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1621	pVmcb->guest.u64CR3 = pCtx->cr3;
1622	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1623	}
1624	else
1625	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1626
1627	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1628	}
1629
1630
1631	/**
1632	* Exports the guest (or nested-guest) CR4 into the VMCB.
1633	*
1634	* @param pVCpu The cross context virtual CPU structure.
1635	* @param pVmcb Pointer to the VM control block.
1636	*
1637	* @remarks No-long-jump zone!!!
1638	*/
1639	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1640	{
1641	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1642
1643	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1644	uint64_t uShadowCr4 = pCtx->cr4;
1645	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1646	{
1647	switch (pVCpu->hm.s.enmShadowMode)
1648	{
1649	case PGMMODE_REAL:
1650	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1651	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1652
1653	case PGMMODE_32_BIT: /* 32-bit paging. */
1654	uShadowCr4 &= ~X86_CR4_PAE;
1655	break;
1656
1657	case PGMMODE_PAE: /* PAE paging. */
1658	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1659	/** Must use PAE paging as we could use physical memory > 4 GB */
1660	uShadowCr4 \|= X86_CR4_PAE;
1661	break;
1662
1663	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1664	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1665	#ifdef VBOX_WITH_64_BITS_GUESTS
1666	break;
1667	#else
1668	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1669	#endif
1670
1671	default: /* shut up gcc */
1672	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1673	}
1674	}
1675
1676	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1677	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1678	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1679	{
1680	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1681	hmR0SvmUpdateVmRunFunction(pVCpu);
1682	}
1683
1684	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1685	if (uShadowCr4 == pCtx->cr4)
1686	{
1687	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1688	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1689	else
1690	{
1691	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1692	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1693	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1694	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1695	}
1696	}
1697	else
1698	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1699
1700	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1701	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1702
1703	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1704	Assert(!RT_HI_U32(uShadowCr4));
1705	pVmcb->guest.u64CR4 = uShadowCr4;
1706	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1707
1708	return VINF_SUCCESS;
1709	}
1710
1711
1712	/**
1713	* Exports the guest (or nested-guest) control registers into the VMCB.
1714	*
1715	* @returns VBox status code.
1716	* @param pVCpu The cross context virtual CPU structure.
1717	* @param pVmcb Pointer to the VM control block.
1718	*
1719	* @remarks No-long-jump zone!!!
1720	*/
1721	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1722	{
1723	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1724
1725	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1726	{
1727	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1728	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1729
1730	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1731	{
1732	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1733	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1734	}
1735
1736	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1737	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1738
1739	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1740	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1741	{
1742	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1743	if (RT_FAILURE(rc))
1744	return rc;
1745	}
1746
1747	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1748	}
1749	return VINF_SUCCESS;
1750	}
1751
1752
1753	/**
1754	* Exports the guest (or nested-guest) segment registers into the VMCB.
1755	*
1756	* @returns VBox status code.
1757	* @param pVCpu The cross context virtual CPU structure.
1758	* @param pVmcb Pointer to the VM control block.
1759	*
1760	* @remarks No-long-jump zone!!!
1761	*/
1762	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1763	{
1764	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1765	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1766
1767	/* Guest segment registers. */
1768	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1769	{
1770	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1771	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1772
1773	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1774	{
1775	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1776	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1777	}
1778
1779	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1780	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1781
1782	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1783	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1784
1785	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1786	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1787
1788	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1789	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1790
1791	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1792	}
1793
1794	/* Guest TR. */
1795	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1796	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1797
1798	/* Guest LDTR. */
1799	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1800	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1801
1802	/* Guest GDTR. */
1803	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1804	{
1805	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1806	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1807	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1808	}
1809
1810	/* Guest IDTR. */
1811	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1812	{
1813	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1814	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1815	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1816	}
1817
1818	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1819	\| HM_CHANGED_GUEST_TABLE_MASK);
1820	}
1821
1822
1823	/**
1824	* Exports the guest (or nested-guest) MSRs into the VMCB.
1825	*
1826	* @param pVCpu The cross context virtual CPU structure.
1827	* @param pVmcb Pointer to the VM control block.
1828	*
1829	* @remarks No-long-jump zone!!!
1830	*/
1831	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1832	{
1833	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1834	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1835
1836	/* Guest Sysenter MSRs. */
1837	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1838	{
1839	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1840	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1841
1842	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1843	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1844
1845	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1846	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1847	}
1848
1849	/*
1850	* Guest EFER MSR.
1851	* AMD-V requires guest EFER.SVME to be set. Weird.
1852	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1853	*/
1854	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1855	{
1856	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1857	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1858	}
1859
1860	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1861	if ( !CPUMIsGuestInLongModeEx(pCtx)
1862	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1863	{
1864	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1865	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1866	}
1867
1868	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1869	{
1870	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1871	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1872	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1873	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1874	}
1875
1876	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1877	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1878
1879	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1880	\| HM_CHANGED_GUEST_EFER_MSR
1881	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1882	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1883
1884	/*
1885	* Setup the PAT MSR (applicable for Nested Paging only).
1886	*
1887	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1888	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1889	*
1890	* While guests can modify and see the modified values through the shadow values,
1891	* we shall not honor any guest modifications of this MSR to ensure caching is always
1892	* enabled similar to how we clear CR0.CD and NW bits.
1893	*
1894	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1895	*/
1896	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1897
1898	/* Enable the last branch record bit if LBR virtualization is enabled. */
1899	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1900	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1901	}
1902
1903
1904	/**
1905	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1906	* the necessary intercepts accordingly.
1907	*
1908	* @param pVCpu The cross context virtual CPU structure.
1909	* @param pVmcb Pointer to the VM control block.
1910	*
1911	* @remarks No-long-jump zone!!!
1912	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1913	*/
1914	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1915	{
1916	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1917
1918	/** @todo Figure out stepping with nested-guest. */
1919	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1920	{
1921	/*
1922	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1923	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1924	* Instead, they are strictly only requested when the nested hypervisor intercepts
1925	* them -- handled while merging VMCB controls.
1926	*
1927	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1928	* then the nested-guest debug state should be actively loaded on the host so that
1929	* nested-guest reads/writes its own debug registers without causing VM-exits.
1930	*/
1931	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1932	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1933	&& !CPUMIsGuestDebugStateActive(pVCpu))
1934	{
1935	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1936	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1937	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1938	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1939	}
1940
1941	pVmcb->guest.u64DR6 = pCtx->dr[6];
1942	pVmcb->guest.u64DR7 = pCtx->dr[7];
1943	return;
1944	}
1945
1946	/*
1947	* Anyone single stepping on the host side? If so, we'll have to use the
1948	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1949	* the VMM level like the VT-x implementations does.
1950	*/
1951	bool fInterceptMovDRx = false;
1952	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1953	if (fStepping)
1954	{
1955	pVCpu->hmr0.s.fClearTrapFlag = true;
1956	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1957	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1958	}
1959
1960	if ( fStepping
1961	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1962	{
1963	/*
1964	* Use the combined guest and host DRx values found in the hypervisor
1965	* register set because the debugger has breakpoints active or someone
1966	* is single stepping on the host side.
1967	*
1968	* Note! DBGF expects a clean DR6 state before executing guest code.
1969	*/
1970	if (!CPUMIsHyperDebugStateActive(pVCpu))
1971	{
1972	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1973	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1974	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1975	}
1976
1977	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1978	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1979	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1980	{
1981	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1982	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1983	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1984	}
1985
1986	/** @todo If we cared, we could optimize to allow the guest to read registers
1987	* with the same values. */
1988	fInterceptMovDRx = true;
1989	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1990	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1991	}
1992	else
1993	{
1994	/*
1995	* Update DR6, DR7 with the guest values if necessary.
1996	*/
1997	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1998	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1999	{
2000	pVmcb->guest.u64DR7 = pCtx->dr[7];
2001	pVmcb->guest.u64DR6 = pCtx->dr[6];
2002	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
2003	}
2004	pVCpu->hmr0.s.fUsingHyperDR7 = false;
2005
2006	/*
2007	* If the guest has enabled debug registers, we need to load them prior to
2008	* executing guest code so they'll trigger at the right time.
2009	*/
2010	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
2011	{
2012	if (!CPUMIsGuestDebugStateActive(pVCpu))
2013	{
2014	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2015	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2016	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2017	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2018	}
2019	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2020	}
2021	/*
2022	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2023	* intercept #DB as DR6 is updated in the VMCB.
2024	*
2025	* Note! If we cared and dared, we could skip intercepting \#DB here.
2026	* However, \#DB shouldn't be performance critical, so we'll play safe
2027	* and keep the code similar to the VT-x code and always intercept it.
2028	*/
2029	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2030	fInterceptMovDRx = true;
2031	}
2032
2033	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2034	if (fInterceptMovDRx)
2035	{
2036	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2037	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2038	{
2039	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2040	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2041	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2042	}
2043	}
2044	else
2045	{
2046	if ( pVmcb->ctrl.u16InterceptRdDRx
2047	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2048	{
2049	pVmcb->ctrl.u16InterceptRdDRx = 0;
2050	pVmcb->ctrl.u16InterceptWrDRx = 0;
2051	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2052	}
2053	}
2054	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2055	}
2056
2057	/**
2058	* Exports the hardware virtualization state into the nested-guest
2059	* VMCB.
2060	*
2061	* @param pVCpu The cross context virtual CPU structure.
2062	* @param pVmcb Pointer to the VM control block.
2063	*
2064	* @remarks No-long-jump zone!!!
2065	*/
2066	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2067	{
2068	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2069
2070	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2071	{
2072	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2073	{
2074	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2075	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2076
2077	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2078	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2079	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2080	NOREF(pVM);
2081
2082	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2083	}
2084
2085	/*
2086	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2087	* since SVM doesn't have a preemption timer.
2088	*
2089	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2090	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2091	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2092	*/
2093	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2094	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2095	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2096	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2097	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2098	{
2099	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2100	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2101	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2102	}
2103	else
2104	{
2105	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2106	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2107	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2108	}
2109	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2110
2111	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2112	}
2113	}
2114
2115
2116	/**
2117	* Exports the guest APIC TPR state into the VMCB.
2118	*
2119	* @returns VBox status code.
2120	* @param pVCpu The cross context virtual CPU structure.
2121	* @param pVmcb Pointer to the VM control block.
2122	*/
2123	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2124	{
2125	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2126
2127	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2128	{
2129	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2130	if ( PDMHasApic(pVM)
2131	&& APICIsEnabled(pVCpu))
2132	{
2133	bool fPendingIntr;
2134	uint8_t u8Tpr;
2135	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2136	AssertRCReturn(rc, rc);
2137
2138	/* Assume that we need to trap all TPR accesses and thus need not check on
2139	every #VMEXIT if we should update the TPR. */
2140	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2141	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2142
2143	if (!pVM->hm.s.fTprPatchingActive)
2144	{
2145	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2146	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2147
2148	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2149	can deliver the interrupt to the guest. */
2150	if (fPendingIntr)
2151	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2152	else
2153	{
2154	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2155	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2156	}
2157
2158	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2159	}
2160	else
2161	{
2162	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2163	pVmcb->guest.u64LSTAR = u8Tpr;
2164	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2165
2166	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2167	if (fPendingIntr)
2168	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2169	else
2170	{
2171	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2172	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2173	}
2174	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2175	}
2176	}
2177	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2178	}
2179	return VINF_SUCCESS;
2180	}
2181
2182
2183	/**
2184	* Sets up the exception interrupts required for guest execution in the VMCB.
2185	*
2186	* @param pVCpu The cross context virtual CPU structure.
2187	* @param pVmcb Pointer to the VM control block.
2188	*
2189	* @remarks No-long-jump zone!!!
2190	*/
2191	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2192	{
2193	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2194
2195	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2196	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2197	{
2198	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2199	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2200	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2201	else
2202	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2203
2204	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2205	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2206	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2207	else
2208	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2209
2210	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2211	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2212	}
2213	}
2214
2215
2216	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2217	/**
2218	* Merges guest and nested-guest intercepts for executing the nested-guest using
2219	* hardware-assisted SVM.
2220	*
2221	* This merges the guest and nested-guest intercepts in a way that if the outer
2222	* guest intercept is set we need to intercept it in the nested-guest as
2223	* well.
2224	*
2225	* @param pVCpu The cross context virtual CPU structure.
2226	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2227	*/
2228	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2229	{
2230	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2231	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2232	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2233	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2234
2235	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2236	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2237	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2238
2239	/* Always intercept CR4 writes for tracking PGM mode changes. */
2240	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2241
2242	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2243	if (!pVM->hmr0.s.fNestedPaging)
2244	{
2245	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2246	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2247	}
2248
2249	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2250	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2251	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2252
2253	/*
2254	* Merge the guest's exception intercepts into the nested-guest VMCB.
2255	*
2256	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2257	* while executing the nested-guest.
2258	*
2259	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2260	* be tweaked later depending on how we wish to implement breakpoints.
2261	*
2262	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2263	* loaded into their guests, not ours.
2264	*
2265	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2266	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2267	*/
2268	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2269	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2270	& ~( RT_BIT(X86_XCPT_UD)
2271	\| RT_BIT(X86_XCPT_BP)
2272	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2273	#else
2274	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2275	#endif
2276
2277	/*
2278	* Adjust intercepts while executing the nested-guest that differ from the
2279	* outer guest intercepts.
2280	*
2281	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2282	* that belong to the nested-guest to the outer guest.
2283	*
2284	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2285	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2286	*/
2287	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2288	\| SVM_CTRL_INTERCEPT_VMMCALL))
2289	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2290
2291	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2292	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2293
2294	/* Finally, update the VMCB clean bits. */
2295	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2296	}
2297	#endif
2298
2299
2300	/**
2301	* Enters the AMD-V session.
2302	*
2303	* @returns VBox status code.
2304	* @param pVCpu The cross context virtual CPU structure.
2305	*/
2306	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2307	{
2308	AssertPtr(pVCpu);
2309	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2310	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2311
2312	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2313	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2314	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2315
2316	pVCpu->hmr0.s.fLeaveDone = false;
2317	return VINF_SUCCESS;
2318	}
2319
2320
2321	/**
2322	* Thread-context callback for AMD-V.
2323	*
2324	* This is used together with RTThreadCtxHookCreate() on platforms which
2325	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2326	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2327	*
2328	* @param enmEvent The thread-context event.
2329	* @param pVCpu The cross context virtual CPU structure.
2330	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2331	* @thread EMT(pVCpu)
2332	*/
2333	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2334	{
2335	NOREF(fGlobalInit);
2336
2337	switch (enmEvent)
2338	{
2339	case RTTHREADCTXEVENT_OUT:
2340	{
2341	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2342	VMCPU_ASSERT_EMT(pVCpu);
2343
2344	/* No longjmps (log-flush, locks) in this fragile context. */
2345	VMMRZCallRing3Disable(pVCpu);
2346
2347	if (!pVCpu->hmr0.s.fLeaveDone)
2348	{
2349	hmR0SvmLeave(pVCpu, false /* fImportState */);
2350	pVCpu->hmr0.s.fLeaveDone = true;
2351	}
2352
2353	/* Leave HM context, takes care of local init (term). */
2354	int rc = HMR0LeaveCpu(pVCpu);
2355	AssertRC(rc); NOREF(rc);
2356
2357	/* Restore longjmp state. */
2358	VMMRZCallRing3Enable(pVCpu);
2359	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2360	break;
2361	}
2362
2363	case RTTHREADCTXEVENT_IN:
2364	{
2365	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2366	VMCPU_ASSERT_EMT(pVCpu);
2367
2368	/* No longjmps (log-flush, locks) in this fragile context. */
2369	VMMRZCallRing3Disable(pVCpu);
2370
2371	/*
2372	* Initialize the bare minimum state required for HM. This takes care of
2373	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2374	*/
2375	int rc = hmR0EnterCpu(pVCpu);
2376	AssertRC(rc); NOREF(rc);
2377	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2378	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2379
2380	pVCpu->hmr0.s.fLeaveDone = false;
2381
2382	/* Restore longjmp state. */
2383	VMMRZCallRing3Enable(pVCpu);
2384	break;
2385	}
2386
2387	default:
2388	break;
2389	}
2390	}
2391
2392
2393	/**
2394	* Saves the host state.
2395	*
2396	* @returns VBox status code.
2397	* @param pVCpu The cross context virtual CPU structure.
2398	*
2399	* @remarks No-long-jump zone!!!
2400	*/
2401	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2402	{
2403	NOREF(pVCpu);
2404
2405	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2406	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2407	return VINF_SUCCESS;
2408	}
2409
2410
2411	/**
2412	* Exports the guest or nested-guest state from the virtual-CPU context into the
2413	* VMCB.
2414	*
2415	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2416	* code based on the virtual-CPU mode.
2417	*
2418	* @returns VBox status code.
2419	* @param pVCpu The cross context virtual CPU structure.
2420	* @param pSvmTransient Pointer to the SVM-transient structure.
2421	*
2422	* @remarks No-long-jump zone!!!
2423	*/
2424	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2425	{
2426	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2427
2428	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2429	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2430	Assert(pVmcb);
2431
2432	pVmcb->guest.u64RIP = pCtx->rip;
2433	pVmcb->guest.u64RSP = pCtx->rsp;
2434	pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2435	pVmcb->guest.u64RAX = pCtx->rax;
2436
2437	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2438	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2439
2440	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2441	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2442	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2443	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2444	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2445
2446	ASMSetFlags(fEFlags);
2447
2448	if (!fIsNestedGuest)
2449	{
2450	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2451	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2452	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2453	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2454	}
2455
2456	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2457	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2458	\| HM_CHANGED_GUEST_RFLAGS
2459	\| HM_CHANGED_GUEST_GPRS_MASK
2460	\| HM_CHANGED_GUEST_X87
2461	\| HM_CHANGED_GUEST_SSE_AVX
2462	\| HM_CHANGED_GUEST_OTHER_XSAVE
2463	\| HM_CHANGED_GUEST_XCRx
2464	\| HM_CHANGED_GUEST_TSC_AUX
2465	\| HM_CHANGED_GUEST_OTHER_MSRS;
2466	if (fIsNestedGuest)
2467	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2468	\| HM_CHANGED_GUEST_APIC_TPR;
2469
2470	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2471	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2472
2473	#ifdef VBOX_STRICT
2474	/*
2475	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2476	* except for the host-context and/or shared host-guest context bits.
2477	*/
2478	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2479	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2480	("fCtxChanged=%#RX64\n", fCtxChanged));
2481
2482	/*
2483	* If we need to log state that isn't always imported, we'll need to import them here.
2484	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2485	*/
2486	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2487	#endif
2488
2489	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2490	return VINF_SUCCESS;
2491	}
2492
2493	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2494
2495	/**
2496	* Merges the guest and nested-guest MSR permission bitmap.
2497	*
2498	* If the guest is intercepting an MSR we need to intercept it regardless of
2499	* whether the nested-guest is intercepting it or not.
2500	*
2501	* @param pHostCpu The HM physical-CPU structure.
2502	* @param pVCpu The cross context virtual CPU structure.
2503	*
2504	* @remarks No-long-jmp zone!!!
2505	*/
2506	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2507	{
2508	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2509	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2510	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2511
2512	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2513	uint32_t const offRsvdQwords = 0x1800 >> 3;
2514	for (uint32_t i = 0; i < offRsvdQwords; i++)
2515	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2516	}
2517
2518
2519	/**
2520	* Caches the nested-guest VMCB fields before we modify them for execution using
2521	* hardware-assisted SVM.
2522	*
2523	* @returns true if the VMCB was previously already cached, false otherwise.
2524	* @param pVCpu The cross context virtual CPU structure.
2525	*
2526	* @sa HMNotifySvmNstGstVmexit.
2527	*/
2528	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2529	{
2530	/*
2531	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2532	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2533	*
2534	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2535	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2536	*/
2537	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2538	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2539	if (!fWasCached)
2540	{
2541	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2542	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2543	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2544	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2545	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2546	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2547	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2548	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2549	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2550	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2551	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2552	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2553	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2554	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2555	pVmcbNstGstCache->fCacheValid = true;
2556	Log4Func(("Cached VMCB fields\n"));
2557	}
2558
2559	return fWasCached;
2560	}
2561
2562
2563	/**
2564	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2565	*
2566	* This is done the first time we enter nested-guest execution using SVM R0
2567	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2568	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2569	*
2570	* @param pVCpu The cross context virtual CPU structure.
2571	*/
2572	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2573	{
2574	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2575	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2576
2577	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2578
2579	/*
2580	* First cache the nested-guest VMCB fields we may potentially modify.
2581	*/
2582	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2583	if (!fVmcbCached)
2584	{
2585	/*
2586	* The IOPM of the nested-guest can be ignored because the the guest always
2587	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2588	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2589	*/
2590	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2591
2592	/*
2593	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2594	* nested-paging suddenly while executing a VM (see assertion at the end of
2595	* Trap0eHandler() in PGMAllBth.h).
2596	*/
2597	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2598
2599	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2600	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2601
2602	/*
2603	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2604	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2605	*/
2606	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2607
2608	# ifdef DEBUG_ramshankar
2609	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2610	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2611	# endif
2612
2613	/*
2614	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2615	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2616	*/
2617	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2618	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2619	\| SVM_CTRL_INTERCEPT_VMLOAD;
2620
2621	/*
2622	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2623	* CLGI/STGI instructions executed by the nested-guest.
2624	*/
2625	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2626	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2627	\| SVM_CTRL_INTERCEPT_STGI;
2628
2629	/* Merge the guest and nested-guest intercepts. */
2630	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2631
2632	/* Update the VMCB clean bits. */
2633	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2634	}
2635	else
2636	{
2637	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2638	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2639	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2640	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2641	}
2642	}
2643
2644	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2645
2646	/**
2647	* Exports the state shared between the host and guest (or nested-guest) into
2648	* the VMCB.
2649	*
2650	* @param pVCpu The cross context virtual CPU structure.
2651	* @param pVmcb Pointer to the VM control block.
2652	*
2653	* @remarks No-long-jump zone!!!
2654	*/
2655	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2656	{
2657	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2658	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2659
2660	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2661	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2662
2663	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2664	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2665	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2666	}
2667
2668
2669	/**
2670	* Worker for SVMR0ImportStateOnDemand.
2671	*
2672	* @param pVCpu The cross context virtual CPU structure.
2673	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2674	*/
2675	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2676	{
2677	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2678
2679	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2680	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2681	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2682	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2683
2684	/*
2685	* We disable interrupts to make the updating of the state and in particular
2686	* the fExtrn modification atomic wrt to preemption hooks.
2687	*/
2688	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2689
2690	fWhat &= pCtx->fExtrn;
2691	if (fWhat)
2692	{
2693	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2694	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2695	{
2696	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2697	{
2698	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2699	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2700	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2701	}
2702	}
2703
2704	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2705	{
2706	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2707	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2708	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2709	}
2710	#endif
2711
2712	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2713	{
2714	if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2715	EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2716	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2717	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2718	}
2719
2720	if (fWhat & CPUMCTX_EXTRN_RIP)
2721	pCtx->rip = pVmcbGuest->u64RIP;
2722
2723	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2724	pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2725
2726	if (fWhat & CPUMCTX_EXTRN_RSP)
2727	pCtx->rsp = pVmcbGuest->u64RSP;
2728
2729	if (fWhat & CPUMCTX_EXTRN_RAX)
2730	pCtx->rax = pVmcbGuest->u64RAX;
2731
2732	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2733	{
2734	if (fWhat & CPUMCTX_EXTRN_CS)
2735	{
2736	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2737	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2738	/** @todo SELM might need to be fixed as it too should not care about the
2739	* granularity bit. See @bugref{6785}. */
2740	if ( !pCtx->cs.Attr.n.u1Granularity
2741	&& pCtx->cs.Attr.n.u1Present
2742	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2743	{
2744	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2745	pCtx->cs.Attr.n.u1Granularity = 1;
2746	}
2747	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2748	}
2749	if (fWhat & CPUMCTX_EXTRN_SS)
2750	{
2751	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2752	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2753	/*
2754	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2755	* VMCB and uses that and thus it's possible that when the CPL changes during
2756	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2757	* AMD Fusion CPUs with 64-bit guests.
2758	*
2759	* See AMD spec. 15.5.1 "Basic operation".
2760	*/
2761	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2762	uint8_t const uCpl = pVmcbGuest->u8CPL;
2763	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2764	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2765	}
2766	if (fWhat & CPUMCTX_EXTRN_DS)
2767	{
2768	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2769	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2770	}
2771	if (fWhat & CPUMCTX_EXTRN_ES)
2772	{
2773	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2774	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2775	}
2776	if (fWhat & CPUMCTX_EXTRN_FS)
2777	{
2778	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2779	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2780	}
2781	if (fWhat & CPUMCTX_EXTRN_GS)
2782	{
2783	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2784	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2785	}
2786	}
2787
2788	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2789	{
2790	if (fWhat & CPUMCTX_EXTRN_TR)
2791	{
2792	/*
2793	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2794	* are used between Intel and AMD, see @bugref{6208#c39}.
2795	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2796	*/
2797	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2798	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2799	{
2800	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2801	\|\| CPUMIsGuestInLongModeEx(pCtx))
2802	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2803	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2804	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2805	}
2806	}
2807
2808	if (fWhat & CPUMCTX_EXTRN_LDTR)
2809	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2810
2811	if (fWhat & CPUMCTX_EXTRN_GDTR)
2812	{
2813	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2814	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2815	}
2816
2817	if (fWhat & CPUMCTX_EXTRN_IDTR)
2818	{
2819	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2820	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2821	}
2822	}
2823
2824	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2825	{
2826	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2827	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2828	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2829	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2830	}
2831
2832	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2833	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2834	{
2835	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2836	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2837	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2838	}
2839
2840	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2841	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2842
2843	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2844	{
2845	if (fWhat & CPUMCTX_EXTRN_DR6)
2846	{
2847	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2848	pCtx->dr[6] = pVmcbGuest->u64DR6;
2849	else
2850	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2851	}
2852
2853	if (fWhat & CPUMCTX_EXTRN_DR7)
2854	{
2855	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2856	pCtx->dr[7] = pVmcbGuest->u64DR7;
2857	else
2858	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2859	}
2860	}
2861
2862	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2863	{
2864	if (fWhat & CPUMCTX_EXTRN_CR0)
2865	{
2866	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2867	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2868	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2869	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2870	CPUMSetGuestCR0(pVCpu, uCr0);
2871	VMMRZCallRing3Enable(pVCpu);
2872	}
2873
2874	if (fWhat & CPUMCTX_EXTRN_CR2)
2875	pCtx->cr2 = pVmcbGuest->u64CR2;
2876
2877	if (fWhat & CPUMCTX_EXTRN_CR3)
2878	{
2879	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2880	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2881	{
2882	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2883	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2884	}
2885	}
2886
2887	/* Changes to CR4 are always intercepted. */
2888	}
2889
2890	/* Update fExtrn. */
2891	pCtx->fExtrn &= ~fWhat;
2892
2893	/* If everything has been imported, clear the HM keeper bit. */
2894	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2895	{
2896	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2897	Assert(!pCtx->fExtrn);
2898	}
2899	}
2900	else
2901	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2902
2903	ASMSetFlags(fEFlags);
2904
2905	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2906
2907	/*
2908	* Honor any pending CR3 updates.
2909	*
2910	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2911	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2912	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2913	* handling -> hmR0SvmImportGuestState() and here we are.
2914	*
2915	* The reason for such complicated handling is because VM-exits that call into PGM expect
2916	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2917	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2918	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2919	*
2920	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2921	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2922	*/
2923	if ( VMMRZCallRing3IsEnabled(pVCpu)
2924	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2925	{
2926	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2927	PGMUpdateCR3(pVCpu, pCtx->cr3);
2928	}
2929	}
2930
2931
2932	/**
2933	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2934	* context.
2935	*
2936	* Currently there is no residual state left in the CPU that is not updated in the
2937	* VMCB.
2938	*
2939	* @returns VBox status code.
2940	* @param pVCpu The cross context virtual CPU structure.
2941	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2942	*/
2943	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2944	{
2945	hmR0SvmImportGuestState(pVCpu, fWhat);
2946	return VINF_SUCCESS;
2947	}
2948
2949
2950	/**
2951	* Does the necessary state syncing before returning to ring-3 for any reason
2952	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2953	*
2954	* @param pVCpu The cross context virtual CPU structure.
2955	* @param fImportState Whether to import the guest state from the VMCB back
2956	* to the guest-CPU context.
2957	*
2958	* @remarks No-long-jmp zone!!!
2959	*/
2960	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2961	{
2962	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2963	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2964
2965	/*
2966	* !!! IMPORTANT !!!
2967	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2968	*/
2969
2970	/* Save the guest state if necessary. */
2971	if (fImportState)
2972	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2973
2974	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2975	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2976	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2977
2978	/*
2979	* Restore host debug registers if necessary and resync on next R0 reentry.
2980	*/
2981	#ifdef VBOX_STRICT
2982	if (CPUMIsHyperDebugStateActive(pVCpu))
2983	{
2984	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2985	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2986	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2987	}
2988	#endif
2989	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2990	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2991	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2992
2993	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2994	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
2995	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
2996	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
2997	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
2998	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
2999	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3000
3001	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3002	}
3003
3004
3005	/**
3006	* Leaves the AMD-V session.
3007	*
3008	* Only used while returning to ring-3 either due to longjump or exits to
3009	* ring-3.
3010	*
3011	* @returns VBox status code.
3012	* @param pVCpu The cross context virtual CPU structure.
3013	*/
3014	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3015	{
3016	HM_DISABLE_PREEMPT(pVCpu);
3017	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3018	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3019
3020	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3021	and done this from the SVMR0ThreadCtxCallback(). */
3022	if (!pVCpu->hmr0.s.fLeaveDone)
3023	{
3024	hmR0SvmLeave(pVCpu, true /* fImportState */);
3025	pVCpu->hmr0.s.fLeaveDone = true;
3026	}
3027
3028	/*
3029	* !!! IMPORTANT !!!
3030	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3031	*/
3032
3033	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3034	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3035	VMMR0ThreadCtxHookDisable(pVCpu);
3036
3037	/* Leave HM context. This takes care of local init (term). */
3038	int rc = HMR0LeaveCpu(pVCpu);
3039
3040	HM_RESTORE_PREEMPT();
3041	return rc;
3042	}
3043
3044
3045	/**
3046	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3047	* any remaining host state) before we go back to ring-3 due to an assertion.
3048	*
3049	* @param pVCpu The cross context virtual CPU structure.
3050	*/
3051	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3052	{
3053	/*
3054	* !!! IMPORTANT !!!
3055	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3056	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3057	*/
3058	VMMR0AssertionRemoveNotification(pVCpu);
3059	VMMRZCallRing3Disable(pVCpu);
3060	HM_DISABLE_PREEMPT(pVCpu);
3061
3062	/* Import the entire guest state. */
3063	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3064
3065	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3066	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3067
3068	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3069	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3070
3071	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3072	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3073	VMMR0ThreadCtxHookDisable(pVCpu);
3074
3075	/* Leave HM context. This takes care of local init (term). */
3076	HMR0LeaveCpu(pVCpu);
3077
3078	HM_RESTORE_PREEMPT();
3079	return VINF_SUCCESS;
3080	}
3081
3082
3083	/**
3084	* Take necessary actions before going back to ring-3.
3085	*
3086	* An action requires us to go back to ring-3. This function does the necessary
3087	* steps before we can safely return to ring-3. This is not the same as longjmps
3088	* to ring-3, this is voluntary.
3089	*
3090	* @returns Strict VBox status code.
3091	* @param pVCpu The cross context virtual CPU structure.
3092	* @param rcExit The reason for exiting to ring-3. Can be
3093	* VINF_VMM_UNKNOWN_RING3_CALL.
3094	*/
3095	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3096	{
3097	Assert(pVCpu);
3098	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3099
3100	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3101	VMMRZCallRing3Disable(pVCpu);
3102	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3103	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3104
3105	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3106	if (pVCpu->hm.s.Event.fPending)
3107	{
3108	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3109	Assert(!pVCpu->hm.s.Event.fPending);
3110	}
3111
3112	/* Sync. the necessary state for going back to ring-3. */
3113	hmR0SvmLeaveSession(pVCpu);
3114	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3115
3116	/* Thread-context hooks are unregistered at this point!!! */
3117	/* Ring-3 callback notifications are unregistered at this point!!! */
3118
3119	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3120	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3121	\| CPUM_CHANGED_LDTR
3122	\| CPUM_CHANGED_GDTR
3123	\| CPUM_CHANGED_IDTR
3124	\| CPUM_CHANGED_TR
3125	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3126	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3127	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3128	{
3129	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3130	}
3131
3132	/* Update the exit-to-ring 3 reason. */
3133	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3134
3135	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3136	if ( rcExit != VINF_EM_RAW_INTERRUPT
3137	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3138	{
3139	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3140	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3141	}
3142
3143	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3144	VMMRZCallRing3Enable(pVCpu);
3145
3146	/*
3147	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3148	* and if we're injecting an event we should have a TRPM trap pending.
3149	*/
3150	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3151	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3152	VERR_SVM_IPE_5);
3153	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3154	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3155	VERR_SVM_IPE_4);
3156
3157	return rcExit;
3158	}
3159
3160
3161	/**
3162	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3163	* intercepts.
3164	*
3165	* @param pVCpu The cross context virtual CPU structure.
3166	* @param pVmcb Pointer to the VM control block.
3167	*
3168	* @remarks No-long-jump zone!!!
3169	*/
3170	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3171	{
3172	/*
3173	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3174	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3175	* RDTSC/RDTSCP as well.
3176	*/
3177	bool fParavirtTsc;
3178	uint64_t uTscOffset;
3179	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3180
3181	bool fIntercept;
3182	if (fCanUseRealTsc)
3183	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3184	else
3185	{
3186	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3187	fIntercept = true;
3188	}
3189
3190	if (!fIntercept)
3191	{
3192	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3193	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3194	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3195	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3196	#endif
3197
3198	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3199	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3200	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3201	}
3202
3203	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3204	information before every VM-entry, hence we have nothing to do here at the moment. */
3205	if (fParavirtTsc)
3206	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3207	}
3208
3209
3210	/**
3211	* Sets an event as a pending event to be injected into the guest.
3212	*
3213	* @param pVCpu The cross context virtual CPU structure.
3214	* @param pEvent Pointer to the SVM event.
3215	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3216	* page-fault.
3217	*
3218	* @remarks Statistics counter assumes this is a guest event being reflected to
3219	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3220	*/
3221	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3222	{
3223	Assert(!pVCpu->hm.s.Event.fPending);
3224	Assert(pEvent->n.u1Valid);
3225
3226	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3227	pVCpu->hm.s.Event.fPending = true;
3228	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3229
3230	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3231	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3232	}
3233
3234
3235	/**
3236	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3237	*
3238	* @param pVCpu The cross context virtual CPU structure.
3239	*/
3240	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3241	{
3242	SVMEVENT Event;
3243	Event.u = 0;
3244	Event.n.u1Valid = 1;
3245	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3246	Event.n.u8Vector = X86_XCPT_UD;
3247	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3248	}
3249
3250
3251	/**
3252	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3253	*
3254	* @param pVCpu The cross context virtual CPU structure.
3255	*/
3256	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3257	{
3258	SVMEVENT Event;
3259	Event.u = 0;
3260	Event.n.u1Valid = 1;
3261	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3262	Event.n.u8Vector = X86_XCPT_DB;
3263	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3264	}
3265
3266
3267	/**
3268	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3269	*
3270	* @param pVCpu The cross context virtual CPU structure.
3271	* @param u32ErrCode The error-code for the page-fault.
3272	* @param uFaultAddress The page fault address (CR2).
3273	*
3274	* @remarks This updates the guest CR2 with @a uFaultAddress!
3275	*/
3276	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3277	{
3278	SVMEVENT Event;
3279	Event.u = 0;
3280	Event.n.u1Valid = 1;
3281	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3282	Event.n.u8Vector = X86_XCPT_PF;
3283	Event.n.u1ErrorCodeValid = 1;
3284	Event.n.u32ErrorCode = u32ErrCode;
3285
3286	/* Update CR2 of the guest. */
3287	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3288	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3289	{
3290	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3291	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3293	}
3294
3295	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3296	}
3297
3298
3299	/**
3300	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3301	*
3302	* @param pVCpu The cross context virtual CPU structure.
3303	*/
3304	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3305	{
3306	SVMEVENT Event;
3307	Event.u = 0;
3308	Event.n.u1Valid = 1;
3309	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3310	Event.n.u8Vector = X86_XCPT_MF;
3311	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3312	}
3313
3314
3315	/**
3316	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3317	*
3318	* @param pVCpu The cross context virtual CPU structure.
3319	*/
3320	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3321	{
3322	SVMEVENT Event;
3323	Event.u = 0;
3324	Event.n.u1Valid = 1;
3325	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3326	Event.n.u8Vector = X86_XCPT_DF;
3327	Event.n.u1ErrorCodeValid = 1;
3328	Event.n.u32ErrorCode = 0;
3329	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3330	}
3331
3332
3333	/**
3334	* Injects an event into the guest upon VMRUN by updating the relevant field
3335	* in the VMCB.
3336	*
3337	* @param pVCpu The cross context virtual CPU structure.
3338	* @param pVmcb Pointer to the guest VM control block.
3339	* @param pEvent Pointer to the event.
3340	*
3341	* @remarks No-long-jump zone!!!
3342	* @remarks Requires CR0!
3343	*/
3344	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3345	{
3346	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3347	pVmcb->ctrl.EventInject.u = pEvent->u;
3348	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3349	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3350	{
3351	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3352	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3353	}
3354	else
3355	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3356	RT_NOREF(pVCpu);
3357
3358	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3359	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3360	}
3361
3362
3363
3364	/**
3365	* Converts any TRPM trap into a pending HM event. This is typically used when
3366	* entering from ring-3 (not longjmp returns).
3367	*
3368	* @param pVCpu The cross context virtual CPU structure.
3369	*/
3370	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3371	{
3372	Assert(TRPMHasTrap(pVCpu));
3373	Assert(!pVCpu->hm.s.Event.fPending);
3374
3375	uint8_t uVector;
3376	TRPMEVENT enmTrpmEvent;
3377	uint32_t uErrCode;
3378	RTGCUINTPTR GCPtrFaultAddress;
3379	uint8_t cbInstr;
3380
3381	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3382	AssertRC(rc);
3383
3384	SVMEVENT Event;
3385	Event.u = 0;
3386	Event.n.u1Valid = 1;
3387	Event.n.u8Vector = uVector;
3388
3389	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3390	if (enmTrpmEvent == TRPM_TRAP)
3391	{
3392	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3393	switch (uVector)
3394	{
3395	case X86_XCPT_NMI:
3396	{
3397	Event.n.u3Type = SVM_EVENT_NMI;
3398	break;
3399	}
3400
3401	case X86_XCPT_BP:
3402	case X86_XCPT_OF:
3403	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3404	RT_FALL_THRU();
3405
3406	case X86_XCPT_PF:
3407	case X86_XCPT_DF:
3408	case X86_XCPT_TS:
3409	case X86_XCPT_NP:
3410	case X86_XCPT_SS:
3411	case X86_XCPT_GP:
3412	case X86_XCPT_AC:
3413	{
3414	Event.n.u1ErrorCodeValid = 1;
3415	Event.n.u32ErrorCode = uErrCode;
3416	break;
3417	}
3418	}
3419	}
3420	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3421	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3422	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3423	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3424	else
3425	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3426
3427	rc = TRPMResetTrap(pVCpu);
3428	AssertRC(rc);
3429
3430	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3431	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3432
3433	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3434	}
3435
3436
3437	/**
3438	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3439	* AMD-V to execute any instruction.
3440	*
3441	* @param pVCpu The cross context virtual CPU structure.
3442	*/
3443	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3444	{
3445	Assert(pVCpu->hm.s.Event.fPending);
3446	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3447
3448	SVMEVENT Event;
3449	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3450
3451	uint8_t uVector = Event.n.u8Vector;
3452	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3453
3454	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3455
3456	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3457	AssertRC(rc);
3458
3459	if (Event.n.u1ErrorCodeValid)
3460	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3461
3462	if ( enmTrapType == TRPM_TRAP
3463	&& uVector == X86_XCPT_PF)
3464	{
3465	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3466	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3467	}
3468	else if (enmTrapType == TRPM_SOFTWARE_INT)
3469	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3470	pVCpu->hm.s.Event.fPending = false;
3471	}
3472
3473
3474	/**
3475	* Checks if the guest (or nested-guest) has an interrupt shadow active right
3476	* now.
3477	*
3478	* @returns @c true if the interrupt shadow is active, @c false otherwise.
3479	* @param pVCpu The cross context virtual CPU structure.
3480	*
3481	* @remarks No-long-jump zone!!!
3482	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3483	*/
3484	static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
3485	{
3486	/*
3487	* Instructions like STI and MOV SS inhibit interrupts till the next instruction
3488	* completes. Check if we should inhibit interrupts or clear any existing
3489	* interrupt inhibition.
3490	*/
3491	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3492	{
3493	if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3494	{
3495	/*
3496	* We can clear the inhibit force flag as even if we go back to the recompiler
3497	* without executing guest code in AMD-V, the flag's condition to be cleared is
3498	* met and thus the cleared state is correct.
3499	*/
3500	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3501	return false;
3502	}
3503	return true;
3504	}
3505	return false;
3506	}
3507
3508
3509	/**
3510	* Sets the virtual interrupt intercept control in the VMCB.
3511	*
3512	* @param pVCpu The cross context virtual CPU structure.
3513	* @param pVmcb Pointer to the VM control block.
3514	*/
3515	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3516	{
3517	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3518
3519	/*
3520	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3521	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3522	* (updating ISR at the right time) and inject the interrupt.
3523	*
3524	* With AVIC is supported, we could make use of the asynchronously delivery without
3525	* #VMEXIT and we would be passing the AVIC page to SVM.
3526	*
3527	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3528	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3529	*/
3530	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3531	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3532	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3533	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3534	Log4(("Set VINTR intercept\n"));
3535	}
3536
3537
3538	/**
3539	* Clears the virtual interrupt intercept control in the VMCB as
3540	* we are figured the guest is unable process any interrupts
3541	* at this point of time.
3542	*
3543	* @param pVCpu The cross context virtual CPU structure.
3544	* @param pVmcb Pointer to the VM control block.
3545	*/
3546	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3547	{
3548	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3549
3550	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3551	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3552	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3553	{
3554	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3555	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3556	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3557	Log4(("Cleared VINTR intercept\n"));
3558	}
3559	}
3560
3561
3562	/**
3563	* Evaluates the event to be delivered to the guest and sets it as the pending
3564	* event.
3565	*
3566	* @returns Strict VBox status code.
3567	* @param pVCpu The cross context virtual CPU structure.
3568	* @param pSvmTransient Pointer to the SVM transient structure.
3569	*/
3570	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3571	{
3572	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3573	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3574	\| CPUMCTX_EXTRN_RFLAGS
3575	\| CPUMCTX_EXTRN_INHIBIT_INT
3576	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3577
3578	Assert(!pVCpu->hm.s.Event.fPending);
3579	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3580	Assert(pVmcb);
3581
3582	bool const fGif = CPUMGetGuestGif(pCtx);
3583	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3584	bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3585
3586	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3587	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3588	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3589
3590	/** @todo SMI. SMIs take priority over NMIs. */
3591
3592	/*
3593	* Check if the guest or nested-guest can receive NMIs.
3594	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3595	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3596	*/
3597	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3598	&& !fBlockNmi)
3599	{
3600	if ( fGif
3601	&& !fIntShadow)
3602	{
3603	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3604	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3605	{
3606	Log4(("Intercepting NMI -> #VMEXIT\n"));
3607	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3608	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3609	}
3610	#endif
3611	Log4(("Setting NMI pending for injection\n"));
3612	SVMEVENT Event;
3613	Event.u = 0;
3614	Event.n.u1Valid = 1;
3615	Event.n.u8Vector = X86_XCPT_NMI;
3616	Event.n.u3Type = SVM_EVENT_NMI;
3617	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3618	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3619	}
3620	else if (!fGif)
3621	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3622	else if (!pSvmTransient->fIsNestedGuest)
3623	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3624	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3625	}
3626	/*
3627	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3628	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3629	* it from the APIC device.
3630	*
3631	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3632	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3633	* do that work when we execute nested-guest code esp. since all the required information
3634	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3635	* the virtual interrupt controller.
3636	*
3637	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3638	*/
3639	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3640	&& !pVCpu->hm.s.fSingleInstruction)
3641	{
3642	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
3643	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3644	if ( fGif
3645	&& !fBlockInt
3646	&& !fIntShadow)
3647	{
3648	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3649	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3650	{
3651	Log4(("Intercepting INTR -> #VMEXIT\n"));
3652	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3653	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3654	}
3655	#endif
3656	uint8_t u8Interrupt;
3657	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3658	if (RT_SUCCESS(rc))
3659	{
3660	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3661	SVMEVENT Event;
3662	Event.u = 0;
3663	Event.n.u1Valid = 1;
3664	Event.n.u8Vector = u8Interrupt;
3665	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3666	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3667	}
3668	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3669	{
3670	/*
3671	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3672	* updated eventually when the TPR is written by the guest.
3673	*/
3674	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3675	}
3676	else
3677	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3678	}
3679	else if (!fGif)
3680	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3681	else if (!pSvmTransient->fIsNestedGuest)
3682	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3683	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3684	}
3685
3686	return VINF_SUCCESS;
3687	}
3688
3689
3690	/**
3691	* Injects any pending events into the guest (or nested-guest).
3692	*
3693	* @param pVCpu The cross context virtual CPU structure.
3694	* @param pVmcb Pointer to the VM control block.
3695	*
3696	* @remarks Must only be called when we are guaranteed to enter
3697	* hardware-assisted SVM execution and not return to ring-3
3698	* prematurely.
3699	*/
3700	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3701	{
3702	Assert(!TRPMHasTrap(pVCpu));
3703	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3704
3705	bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3706	#ifdef VBOX_STRICT
3707	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3708	bool const fGif = CPUMGetGuestGif(pCtx);
3709	bool fAllowInt = fGif;
3710	if (fGif)
3711	{
3712	/*
3713	* For nested-guests we have no way to determine if we're injecting a physical or
3714	* virtual interrupt at this point. Hence the partial verification below.
3715	*/
3716	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3717	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3718	else
3719	fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3720	}
3721	#endif
3722
3723	if (pVCpu->hm.s.Event.fPending)
3724	{
3725	SVMEVENT Event;
3726	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3727	Assert(Event.n.u1Valid);
3728
3729	/*
3730	* Validate event injection pre-conditions.
3731	*/
3732	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3733	{
3734	Assert(fAllowInt);
3735	Assert(!fIntShadow);
3736	}
3737	else if (Event.n.u3Type == SVM_EVENT_NMI)
3738	{
3739	Assert(fGif);
3740	Assert(!fIntShadow);
3741	}
3742
3743	/*
3744	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3745	* do this only when we are surely going to inject the NMI as otherwise if we return
3746	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3747	* SVM R0.
3748	*
3749	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3750	* which will set the VMCS field after actually delivering the NMI which we read on
3751	* VM-exit to determine the state.
3752	*/
3753	if ( Event.n.u3Type == SVM_EVENT_NMI
3754	&& Event.n.u8Vector == X86_XCPT_NMI
3755	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3756	{
3757	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3758	}
3759
3760	/*
3761	* Inject it (update VMCB for injection by the hardware).
3762	*/
3763	Log4(("Injecting pending HM event\n"));
3764	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3765	pVCpu->hm.s.Event.fPending = false;
3766
3767	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3768	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3769	else
3770	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3771	}
3772	else
3773	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3774
3775	/*
3776	* We could have injected an NMI through IEM and continue guest execution using
3777	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3778	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3779	*/
3780	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3781	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3782
3783	/*
3784	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3785	*
3786	* For nested-guests: We need to update it too for the scenario where IEM executes
3787	* the nested-guest but execution later continues here with an interrupt shadow active.
3788	*/
3789	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3790	}
3791
3792
3793	/**
3794	* Reports world-switch error and dumps some useful debug info.
3795	*
3796	* @param pVCpu The cross context virtual CPU structure.
3797	* @param rcVMRun The return code from VMRUN (or
3798	* VERR_SVM_INVALID_GUEST_STATE for invalid
3799	* guest-state).
3800	*/
3801	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3802	{
3803	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3804	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3805	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3806
3807	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3808	{
3809	#ifdef VBOX_STRICT
3810	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3811	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3812	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3813	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3814	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3815	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3816	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3817	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3818	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3819	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3820	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3821	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3822
3823	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3824	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3825	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3826
3827	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3828	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3829	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3830	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3831	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3832	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3833	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3834	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3835	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3836	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3837	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3838	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3839
3840	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3841	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3842	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3843	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3844	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3845	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3846	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3847	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3848	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3849	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3850	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3851	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3852	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3853	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3854	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3855	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3856	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3857	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3858	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3859	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3860
3861	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3862
3863	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3864	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3865
3866	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3867	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3868	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3869	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3870	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3871	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3872	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3873	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3874	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3875	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3876	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3877	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3878	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3879	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3880	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3881	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3882	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3883	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3884	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3885	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3886
3887	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3888	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3889
3890	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3891	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3892	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3893	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3894
3895	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3896	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3897
3898	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3899	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3900	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3901	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3902
3903	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3904	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3905	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3906	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3907	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3908	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3909	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3910
3911	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3912	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3913	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3914	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3915
3916	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3917	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3918	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3919
3920	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3921	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3922	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3923	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3924	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3925	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3926	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3927	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3928	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3929	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3930	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3931	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3932
3933	NOREF(pVmcb);
3934	#endif /* VBOX_STRICT */
3935	}
3936	else
3937	Log4Func(("rcVMRun=%d\n", rcVMRun));
3938	}
3939
3940
3941	/**
3942	* Check per-VM and per-VCPU force flag actions that require us to go back to
3943	* ring-3 for one reason or another.
3944	*
3945	* @returns Strict VBox status code (information status code included).
3946	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3947	* ring-3.
3948	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3949	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3950	* interrupts)
3951	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3952	* all EMTs to be in ring-3.
3953	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3954	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3955	* to the EM loop.
3956	*
3957	* @param pVCpu The cross context virtual CPU structure.
3958	*/
3959	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3960	{
3961	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3962
3963	/* Could happen as a result of longjump. */
3964	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3965	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3966
3967	/* Update pending interrupts into the APIC's IRR. */
3968	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3969	APICUpdatePendingInterrupts(pVCpu);
3970
3971	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3972	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3973	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3974	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3975	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3976	{
3977	/* Pending PGM C3 sync. */
3978	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3979	{
3980	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3981	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3982	if (rc != VINF_SUCCESS)
3983	{
3984	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3985	return rc;
3986	}
3987	}
3988
3989	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3990	/* -XXX- what was that about single stepping? */
3991	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3992	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3993	{
3994	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3995	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3996	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3997	return rc;
3998	}
3999
4000	/* Pending VM request packets, such as hardware interrupts. */
4001	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4002	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4003	{
4004	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
4005	Log4Func(("Pending VM request forcing us back to ring-3\n"));
4006	return VINF_EM_PENDING_REQUEST;
4007	}
4008
4009	/* Pending PGM pool flushes. */
4010	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4011	{
4012	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4013	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4014	return VINF_PGM_POOL_FLUSH_PENDING;
4015	}
4016
4017	/* Pending DMA requests. */
4018	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4019	{
4020	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4021	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4022	return VINF_EM_RAW_TO_R3;
4023	}
4024	}
4025
4026	return VINF_SUCCESS;
4027	}
4028
4029
4030	/**
4031	* Does the preparations before executing guest code in AMD-V.
4032	*
4033	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4034	* recompiler. We must be cautious what we do here regarding committing
4035	* guest-state information into the VMCB assuming we assuredly execute the guest
4036	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4037	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4038	* that the recompiler can (and should) use them when it resumes guest
4039	* execution. Otherwise such operations must be done when we can no longer
4040	* exit to ring-3.
4041	*
4042	* @returns Strict VBox status code (informational status codes included).
4043	* @retval VINF_SUCCESS if we can proceed with running the guest.
4044	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4045	*
4046	* @param pVCpu The cross context virtual CPU structure.
4047	* @param pSvmTransient Pointer to the SVM transient structure.
4048	*/
4049	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4050	{
4051	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4052
4053	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4054	if (pSvmTransient->fIsNestedGuest)
4055	{
4056	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4057	return VINF_EM_RESCHEDULE_REM;
4058	}
4059	#endif
4060
4061	/* Check force flag actions that might require us to go back to ring-3. */
4062	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4063	if (rc != VINF_SUCCESS)
4064	return rc;
4065
4066	if (TRPMHasTrap(pVCpu))
4067	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4068	else if (!pVCpu->hm.s.Event.fPending)
4069	{
4070	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4071	if ( rc != VINF_SUCCESS
4072	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4073	{
4074	/* If a nested-guest VM-exit occurred, bail. */
4075	if (pSvmTransient->fIsNestedGuest)
4076	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4077	return rc;
4078	}
4079	}
4080
4081	/*
4082	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4083	* Just do it in software, see @bugref{8411}.
4084	* NB: If we could continue a task switch exit we wouldn't need to do this.
4085	*/
4086	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4087	if (RT_UNLIKELY( !g_fHmSvmFeatures
4088	&& pVCpu->hm.s.Event.fPending
4089	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4090	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4091
4092	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4093	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4094	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4095	#endif
4096
4097	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4098	/*
4099	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4100	*/
4101	if (pSvmTransient->fIsNestedGuest)
4102	hmR0SvmSetupVmcbNested(pVCpu);
4103	#endif
4104
4105	/*
4106	* Export the guest state bits that are not shared with the host in any way as we can
4107	* longjmp or get preempted in the midst of exporting some of the state.
4108	*/
4109	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4110	AssertRCReturn(rc, rc);
4111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4112
4113	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4114	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4115
4116	/*
4117	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4118	* world-switch so we can update it on the way back if the guest changed the TPR.
4119	*/
4120	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4121	{
4122	Assert(!pSvmTransient->fIsNestedGuest);
4123	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4124	if (pVM->hm.s.fTprPatchingActive)
4125	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4126	else
4127	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4128	}
4129
4130	/*
4131	* No longjmps to ring-3 from this point on!!!
4132	*
4133	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4134	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4135	*/
4136	VMMRZCallRing3Disable(pVCpu);
4137
4138	/*
4139	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4140	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4141	* preemption disabled for a while. Since this is purly to aid the
4142	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4143	* disable interrupt on NT.
4144	*
4145	* We need to check for force-flags that could've possible been altered since we last
4146	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4147	* see @bugref{6398}).
4148	*
4149	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4150	* to ring-3 before executing guest code.
4151	*/
4152	pSvmTransient->fEFlags = ASMIntDisableFlags();
4153	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4154	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4155	{
4156	ASMSetFlags(pSvmTransient->fEFlags);
4157	VMMRZCallRing3Enable(pVCpu);
4158	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4159	return VINF_EM_RAW_TO_R3;
4160	}
4161	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4162	{
4163	ASMSetFlags(pSvmTransient->fEFlags);
4164	VMMRZCallRing3Enable(pVCpu);
4165	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4166	return VINF_EM_RAW_INTERRUPT;
4167	}
4168
4169	return VINF_SUCCESS;
4170	}
4171
4172
4173	/**
4174	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4175	* doing so.
4176	*
4177	* This means there is no backing out to ring-3 or anywhere else at this point.
4178	*
4179	* @param pVCpu The cross context virtual CPU structure.
4180	* @param pSvmTransient Pointer to the SVM transient structure.
4181	*
4182	* @remarks Called with preemption disabled.
4183	* @remarks No-long-jump zone!!!
4184	*/
4185	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4186	{
4187	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4188	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4189
4190	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4191	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4192
4193	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4194	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4195
4196	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4197
4198	if (!CPUMIsGuestFPUStateActive(pVCpu))
4199	{
4200	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4201	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4202	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4203	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4204	}
4205
4206	/* Load the state shared between host and guest (FPU, debug). */
4207	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4208	hmR0SvmExportSharedState(pVCpu, pVmcb);
4209
4210	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4211	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4212
4213	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4214	RTCPUID const idHostCpu = pHostCpu->idCpu;
4215	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4216
4217	/* Setup TSC offsetting. */
4218	if ( pSvmTransient->fUpdateTscOffsetting
4219	\|\| fMigratedHostCpu)
4220	{
4221	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4222	pSvmTransient->fUpdateTscOffsetting = false;
4223	}
4224
4225	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4226	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4227	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4228	else
4229	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4230
4231	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4232	if (fMigratedHostCpu)
4233	pVmcb->ctrl.u32VmcbCleanBits = 0;
4234
4235	/* Store status of the shared guest-host state at the time of VMRUN. */
4236	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4237	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4238
4239	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4240	uint8_t *pbMsrBitmap;
4241	if (!pSvmTransient->fIsNestedGuest)
4242	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4243	else
4244	{
4245	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4246	* MSRPM and only perform this if it changed also use EVEX.POR when it
4247	* does. */
4248	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4249
4250	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4251	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4252	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4253	}
4254	#else
4255	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4256	#endif
4257
4258	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4259	/* Flush the appropriate tagged-TLB entries. */
4260	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4261	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4262
4263	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4264
4265	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4266	to start executing. */
4267
4268	/*
4269	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4270	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4271	*
4272	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4273	*/
4274	if ( pVM->cpum.ro.HostFeatures.fRdTscP
4275	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4276	{
4277	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4278	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4279	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4280	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4281	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4282	pSvmTransient->fRestoreTscAuxMsr = true;
4283	}
4284	else
4285	{
4286	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4287	pSvmTransient->fRestoreTscAuxMsr = false;
4288	}
4289	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4290
4291	/*
4292	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4293	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4294	* from the VMCB.
4295	*/
4296	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4297	if (!fSupportsVmcbCleanBits)
4298	pVmcb->ctrl.u32VmcbCleanBits = 0;
4299	}
4300
4301
4302	/**
4303	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4304	*
4305	* @returns VBox strict status code.
4306	* @param pVCpu The cross context virtual CPU structure.
4307	* @param HCPhysVmcb The host physical address of the VMCB.
4308	*
4309	* @remarks No-long-jump zone!!!
4310	*/
4311	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4312	{
4313	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4314	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4315	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4316	}
4317
4318
4319	/**
4320	* Performs some essential restoration of state after running guest (or
4321	* nested-guest) code in AMD-V.
4322	*
4323	* @param pVCpu The cross context virtual CPU structure.
4324	* @param pSvmTransient Pointer to the SVM transient structure.
4325	* @param rcVMRun Return code of VMRUN.
4326	*
4327	* @remarks Called with interrupts disabled.
4328	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4329	* unconditionally when it is safe to do so.
4330	*/
4331	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4332	{
4333	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4334
4335	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4336	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4337
4338	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4339	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4340
4341	/* TSC read must be done early for maximum accuracy. */
4342	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4343	{
4344	if (!pSvmTransient->fIsNestedGuest)
4345	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4346	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4347	else
4348	{
4349	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4350	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4351	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4352	}
4353	#endif
4354	}
4355
4356	if (pSvmTransient->fRestoreTscAuxMsr)
4357	{
4358	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4359	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4360	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4361	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4362	}
4363
4364	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4365	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4366	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4367	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4368
4369	Assert(!(ASMGetFlags() & X86_EFL_IF));
4370	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4371	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4372
4373	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4374	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4375	{
4376	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4377	return;
4378	}
4379
4380	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4381	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4382	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4383	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4384
4385	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4386	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4387	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4388	#else
4389	/*
4390	* Always import the following:
4391	*
4392	* - RIP for exit optimizations and evaluating event injection on re-entry.
4393	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4394	* state on preemption.
4395	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4396	* - CS for exit optimizations.
4397	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4398	* assembly switcher code.
4399	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4400	*/
4401	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4402	\| CPUMCTX_EXTRN_RFLAGS
4403	\| CPUMCTX_EXTRN_RAX
4404	\| CPUMCTX_EXTRN_RSP
4405	\| CPUMCTX_EXTRN_CS
4406	\| CPUMCTX_EXTRN_HWVIRT
4407	\| CPUMCTX_EXTRN_INHIBIT_INT
4408	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4409	\| HMSVM_CPUMCTX_SHARED_STATE);
4410	#endif
4411
4412	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4413	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4414	{
4415	Assert(!pSvmTransient->fIsNestedGuest);
4416	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4417	if ( pVM->hm.s.fTprPatchingActive
4418	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4419	{
4420	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4421	AssertRC(rc);
4422	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4423	}
4424	/* Sync TPR when we aren't intercepting CR8 writes. */
4425	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4426	{
4427	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4428	AssertRC(rc);
4429	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4430	}
4431	}
4432
4433	#ifdef DEBUG_ramshankar
4434	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4435	{
4436	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4437	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4438	0 /* uVerbose */);
4439	}
4440	#endif
4441
4442	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4443	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4444	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4445	}
4446
4447
4448	/**
4449	* Runs the guest code using AMD-V.
4450	*
4451	* @returns Strict VBox status code.
4452	* @param pVCpu The cross context virtual CPU structure.
4453	* @param pcLoops Pointer to the number of executed loops.
4454	*/
4455	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4456	{
4457	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4458	Assert(pcLoops);
4459	Assert(*pcLoops <= cMaxResumeLoops);
4460
4461	SVMTRANSIENT SvmTransient;
4462	RT_ZERO(SvmTransient);
4463	SvmTransient.fUpdateTscOffsetting = true;
4464	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4465
4466	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4467	for (;;)
4468	{
4469	Assert(!HMR0SuspendPending());
4470	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4471
4472	/* Preparatory work for running nested-guest code, this may force us to return to
4473	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4474	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4475	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4476	if (rc != VINF_SUCCESS)
4477	break;
4478
4479	/*
4480	* No longjmps to ring-3 from this point on!!!
4481	*
4482	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4483	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4484	*/
4485	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4486	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4487
4488	/* Restore any residual host-state and save any bits shared between host and guest
4489	into the guest-CPU state. Re-enables interrupts! */
4490	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4491
4492	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4493	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4494	{
4495	if (rc == VINF_SUCCESS)
4496	rc = VERR_SVM_INVALID_GUEST_STATE;
4497	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4498	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4499	break;
4500	}
4501
4502	/* Handle the #VMEXIT. */
4503	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4504	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4505	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4506	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4507	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4508	if (rc != VINF_SUCCESS)
4509	break;
4510	if (++(*pcLoops) >= cMaxResumeLoops)
4511	{
4512	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4513	rc = VINF_EM_RAW_INTERRUPT;
4514	break;
4515	}
4516	}
4517
4518	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4519	return rc;
4520	}
4521
4522
4523	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4524	/**
4525	* Runs the nested-guest code using AMD-V.
4526	*
4527	* @returns Strict VBox status code.
4528	* @param pVCpu The cross context virtual CPU structure.
4529	* @param pcLoops Pointer to the number of executed loops. If we're switching
4530	* from the guest-code execution loop to this nested-guest
4531	* execution loop pass the remainder value, else pass 0.
4532	*/
4533	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4534	{
4535	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4536	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4537	Assert(pcLoops);
4538	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4539	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4540	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4541
4542	SVMTRANSIENT SvmTransient;
4543	RT_ZERO(SvmTransient);
4544	SvmTransient.fUpdateTscOffsetting = true;
4545	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4546	SvmTransient.fIsNestedGuest = true;
4547
4548	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4549	for (;;)
4550	{
4551	Assert(!HMR0SuspendPending());
4552	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4553
4554	/* Preparatory work for running nested-guest code, this may force us to return to
4555	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4556	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4557	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4558	if ( rc != VINF_SUCCESS
4559	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4560	break;
4561
4562	/*
4563	* No longjmps to ring-3 from this point on!!!
4564	*
4565	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4566	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4567	*/
4568	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4569
4570	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4571
4572	/* Restore any residual host-state and save any bits shared between host and guest
4573	into the guest-CPU state. Re-enables interrupts! */
4574	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4575
4576	if (RT_LIKELY( rc == VINF_SUCCESS
4577	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4578	{ /* extremely likely */ }
4579	else
4580	{
4581	/* VMRUN failed, shouldn't really happen, Guru. */
4582	if (rc != VINF_SUCCESS)
4583	break;
4584
4585	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4586	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4587	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4588	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4589	break;
4590	}
4591
4592	/* Handle the #VMEXIT. */
4593	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4594	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4595	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4596	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4597	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4598	if (rc == VINF_SUCCESS)
4599	{
4600	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4601	{
4602	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4603	rc = VINF_SVM_VMEXIT;
4604	}
4605	else
4606	{
4607	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4608	continue;
4609	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4610	rc = VINF_EM_RAW_INTERRUPT;
4611	}
4612	}
4613	else
4614	Assert(rc != VINF_SVM_VMEXIT);
4615	break;
4616	/** @todo NSTSVM: handle single-stepping. */
4617	}
4618
4619	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4620	return rc;
4621	}
4622	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4623
4624
4625	/**
4626	* Checks if any expensive dtrace probes are enabled and we should go to the
4627	* debug loop.
4628	*
4629	* @returns true if we should use debug loop, false if not.
4630	*/
4631	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4632	{
4633	/* It's probably faster to OR the raw 32-bit counter variables together.
4634	Since the variables are in an array and the probes are next to one
4635	another (more or less), we have good locality. So, better read
4636	eight-nine cache lines ever time and only have one conditional, than
4637	128+ conditionals, right? */
4638	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4639	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4651	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4652	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4653	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4654	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4655	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4656	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4657	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4658	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4659	) != 0
4660	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4680	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4682	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4684	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4685	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4686	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4688	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4689	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4690	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4691	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4692	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4693	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4694	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4695	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4696	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4697	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4698	) != 0
4699	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4720	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4722	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4724	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4725	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4726	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4728	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4729	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4730	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4731	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4732	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4733	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4734	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4735	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4736	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4737	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4738	) != 0;
4739	}
4740
4741
4742	/**
4743	* Runs the guest code using AMD-V.
4744	*
4745	* @returns Strict VBox status code.
4746	* @param pVCpu The cross context virtual CPU structure.
4747	*/
4748	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4749	{
4750	AssertPtr(pVCpu);
4751	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4752	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4753	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4754	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4755
4756	uint32_t cLoops = 0;
4757	VBOXSTRICTRC rc;
4758	for (;;)
4759	{
4760	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4761	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4762	#else
4763	NOREF(pCtx);
4764	bool const fInNestedGuestMode = false;
4765	#endif
4766	if (!fInNestedGuestMode)
4767	{
4768	if ( !pVCpu->hm.s.fUseDebugLoop
4769	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4770	&& !DBGFIsStepping(pVCpu)
4771	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4772	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4773	else
4774	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4775	}
4776	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4777	else
4778	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4779
4780	if (rc == VINF_SVM_VMRUN)
4781	{
4782	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4783	continue;
4784	}
4785	if (rc == VINF_SVM_VMEXIT)
4786	{
4787	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4788	continue;
4789	}
4790	#endif
4791	break;
4792	}
4793
4794	/* Fixup error codes. */
4795	if (rc == VERR_EM_INTERPRETER)
4796	rc = VINF_EM_RAW_EMULATE_INSTR;
4797	else if (rc == VINF_EM_RESET)
4798	rc = VINF_EM_TRIPLE_FAULT;
4799
4800	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4801	rc = hmR0SvmExitToRing3(pVCpu, rc);
4802	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4803	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4804	return rc;
4805	}
4806
4807	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4808
4809	/**
4810	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4811	*
4812	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4813	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4814	*/
4815	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4816	{
4817	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4818	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4819	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4820	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4821	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4822	const bool fRep = pIoExitInfo->n.u1Rep;
4823	const bool fStrIo = pIoExitInfo->n.u1Str;
4824
4825	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4826	NULL /* pIoExitInfo */);
4827	}
4828
4829
4830	/**
4831	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4832	* SVM_EXIT_INVALID).
4833	*
4834	* @returns VBox status code (informational status codes included).
4835	* @param pVCpu The cross context virtual CPU structure.
4836	* @param pSvmTransient Pointer to the SVM transient structure.
4837	*/
4838	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4839	{
4840	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4841	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4842	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4843
4844	/*
4845	* We import the complete state here because we use separate VMCBs for the guest and the
4846	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4847	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4848	*/
4849	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4850	do { \
4851	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4852	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4853	} while (0)
4854
4855	/*
4856	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4857	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4858	*/
4859	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4860	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4861	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4862	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4863	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4864	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4865
4866	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4867	switch (uExitCode)
4868	{
4869	case SVM_EXIT_CPUID:
4870	{
4871	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4872	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4873	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4874	}
4875
4876	case SVM_EXIT_RDTSC:
4877	{
4878	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4879	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4880	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4881	}
4882
4883	case SVM_EXIT_RDTSCP:
4884	{
4885	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4886	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4887	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4888	}
4889
4890	case SVM_EXIT_MONITOR:
4891	{
4892	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4893	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4894	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4895	}
4896
4897	case SVM_EXIT_MWAIT:
4898	{
4899	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4900	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4901	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4902	}
4903
4904	case SVM_EXIT_HLT:
4905	{
4906	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4907	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4908	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4909	}
4910
4911	case SVM_EXIT_MSR:
4912	{
4913	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4914	{
4915	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4916	uint16_t offMsrpm;
4917	uint8_t uMsrpmBit;
4918	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4919	if (RT_SUCCESS(rc))
4920	{
4921	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4922	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4923
4924	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4925	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4926	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4927
4928	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4929	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4930	{
4931	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4932	}
4933	}
4934	else
4935	{
4936	/*
4937	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4938	* See AMD-V spec. "15.11 MSR Intercepts".
4939	*/
4940	Assert(rc == VERR_OUT_OF_RANGE);
4941	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4942	}
4943	}
4944	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4945	}
4946
4947	case SVM_EXIT_IOIO:
4948	{
4949	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4950	{
4951	SVMIOIOEXITINFO IoExitInfo;
4952	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4953	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4954	if (fIntercept)
4955	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4956	}
4957	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4958	}
4959
4960	case SVM_EXIT_XCPT_PF:
4961	{
4962	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4963	if (pVM->hmr0.s.fNestedPaging)
4964	{
4965	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4966	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4967
4968	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4969	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4970	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4971
4972	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4973	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4974	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4975	return VINF_SUCCESS;
4976	}
4977	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4978	}
4979
4980	case SVM_EXIT_XCPT_UD:
4981	{
4982	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4983	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4984	hmR0SvmSetPendingXcptUD(pVCpu);
4985	return VINF_SUCCESS;
4986	}
4987
4988	case SVM_EXIT_XCPT_MF:
4989	{
4990	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4991	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4992	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4993	}
4994
4995	case SVM_EXIT_XCPT_DB:
4996	{
4997	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4998	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4999	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5000	}
5001
5002	case SVM_EXIT_XCPT_AC:
5003	{
5004	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5005	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5006	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5007	}
5008
5009	case SVM_EXIT_XCPT_BP:
5010	{
5011	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5012	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5013	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5014	}
5015
5016	case SVM_EXIT_READ_CR0:
5017	case SVM_EXIT_READ_CR3:
5018	case SVM_EXIT_READ_CR4:
5019	{
5020	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5021	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5022	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5023	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5024	}
5025
5026	case SVM_EXIT_CR0_SEL_WRITE:
5027	{
5028	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5029	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5030	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5031	}
5032
5033	case SVM_EXIT_WRITE_CR0:
5034	case SVM_EXIT_WRITE_CR3:
5035	case SVM_EXIT_WRITE_CR4:
5036	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5037	{
5038	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5039	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5040
5041	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5042	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5043	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5044	}
5045
5046	case SVM_EXIT_PAUSE:
5047	{
5048	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5049	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5050	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5051	}
5052
5053	case SVM_EXIT_VINTR:
5054	{
5055	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5056	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5057	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5058	}
5059
5060	case SVM_EXIT_INTR:
5061	case SVM_EXIT_NMI:
5062	case SVM_EXIT_SMI:
5063	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5064	{
5065	/*
5066	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5067	*
5068	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5069	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5070	* nested-guest #VMEXIT.
5071	*
5072	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5073	* while trying to inject interrupts, see comment at the top of this function.
5074	*/
5075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5076	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5077	}
5078
5079	case SVM_EXIT_FERR_FREEZE:
5080	{
5081	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5082	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5083	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5084	}
5085
5086	case SVM_EXIT_INVLPG:
5087	{
5088	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5089	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5090	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5091	}
5092
5093	case SVM_EXIT_WBINVD:
5094	{
5095	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5096	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5097	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5098	}
5099
5100	case SVM_EXIT_INVD:
5101	{
5102	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5103	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5104	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5105	}
5106
5107	case SVM_EXIT_RDPMC:
5108	{
5109	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5110	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5111	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5112	}
5113
5114	default:
5115	{
5116	switch (uExitCode)
5117	{
5118	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5119	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5120	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5121	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5122	{
5123	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5124	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5125	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5126	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5127	}
5128
5129	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5130	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5131	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5132	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5133	{
5134	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5135	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5136	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5137	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5138	}
5139
5140	case SVM_EXIT_XCPT_DE:
5141	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5142	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5143	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5144	case SVM_EXIT_XCPT_OF:
5145	case SVM_EXIT_XCPT_BR:
5146	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5147	case SVM_EXIT_XCPT_NM:
5148	case SVM_EXIT_XCPT_DF:
5149	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5150	case SVM_EXIT_XCPT_TS:
5151	case SVM_EXIT_XCPT_NP:
5152	case SVM_EXIT_XCPT_SS:
5153	case SVM_EXIT_XCPT_GP:
5154	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5155	case SVM_EXIT_XCPT_15: /* Reserved. */
5156	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5157	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5158	case SVM_EXIT_XCPT_MC:
5159	case SVM_EXIT_XCPT_XF:
5160	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5161	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5162	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5163	{
5164	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5165	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5166	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5167	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5168	}
5169
5170	case SVM_EXIT_XSETBV:
5171	{
5172	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5173	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5174	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5175	}
5176
5177	case SVM_EXIT_TASK_SWITCH:
5178	{
5179	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5180	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5181	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5182	}
5183
5184	case SVM_EXIT_IRET:
5185	{
5186	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5187	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5188	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5189	}
5190
5191	case SVM_EXIT_SHUTDOWN:
5192	{
5193	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5194	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5195	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5196	}
5197
5198	case SVM_EXIT_VMMCALL:
5199	{
5200	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5201	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5202	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5203	}
5204
5205	case SVM_EXIT_CLGI:
5206	{
5207	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5208	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5209	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5210	}
5211
5212	case SVM_EXIT_STGI:
5213	{
5214	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5215	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5216	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5217	}
5218
5219	case SVM_EXIT_VMLOAD:
5220	{
5221	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5222	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5223	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5224	}
5225
5226	case SVM_EXIT_VMSAVE:
5227	{
5228	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5229	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5230	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5231	}
5232
5233	case SVM_EXIT_INVLPGA:
5234	{
5235	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5236	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5237	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5238	}
5239
5240	case SVM_EXIT_VMRUN:
5241	{
5242	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5243	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5244	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5245	}
5246
5247	case SVM_EXIT_RSM:
5248	{
5249	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5250	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5251	hmR0SvmSetPendingXcptUD(pVCpu);
5252	return VINF_SUCCESS;
5253	}
5254
5255	case SVM_EXIT_SKINIT:
5256	{
5257	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5258	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5259	hmR0SvmSetPendingXcptUD(pVCpu);
5260	return VINF_SUCCESS;
5261	}
5262
5263	case SVM_EXIT_NPF:
5264	{
5265	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5266	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5267	}
5268
5269	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5270	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5271
5272	default:
5273	{
5274	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5275	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5276	return VERR_SVM_UNKNOWN_EXIT;
5277	}
5278	}
5279	}
5280	}
5281	/* not reached */
5282
5283	# undef NST_GST_VMEXIT_CALL_RET
5284	}
5285
5286	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5287
5288	/** @def VMEXIT_CALL_RET
5289	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5290	*/
5291	#ifdef DEBUG_ramshankar
5292	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5293	do { \
5294	if ((a_fDbg) == 1) \
5295	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5296	int rc = a_CallExpr; \
5297	if ((a_fDbg) == 1) \
5298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5299	return rc; \
5300	} while (0)
5301	#else
5302	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5303	#endif
5304
5305	/**
5306	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5307	*
5308	* @returns Strict VBox status code (informational status codes included).
5309	* @param pVCpu The cross context virtual CPU structure.
5310	* @param pSvmTransient Pointer to the SVM transient structure.
5311	*/
5312	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5313	{
5314	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5315	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5316
5317	/*
5318	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5319	* for most guests under normal workloads (for some definition of "normal").
5320	*/
5321	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5322	switch (uExitCode)
5323	{
5324	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5325	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5326	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5327	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5328	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5329	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5330	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5331	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5332	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5333	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5334
5335	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5336	case SVM_EXIT_INTR:
5337	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5338
5339	case SVM_EXIT_READ_CR0:
5340	case SVM_EXIT_READ_CR3:
5341	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5342
5343	case SVM_EXIT_CR0_SEL_WRITE:
5344	case SVM_EXIT_WRITE_CR0:
5345	case SVM_EXIT_WRITE_CR3:
5346	case SVM_EXIT_WRITE_CR4:
5347	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5348
5349	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5350	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5351	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5352	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5353	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5354	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5355	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5356	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5357	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5358	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5359	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5360	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5361	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5362	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5363	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5364	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5365
5366	default:
5367	{
5368	switch (pSvmTransient->u64ExitCode)
5369	{
5370	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5371	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5372	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5373	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5374	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5375
5376	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5377	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5378	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5379	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5380	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5381
5382	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5383	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5384
5385	case SVM_EXIT_SMI:
5386	case SVM_EXIT_INIT:
5387	{
5388	/*
5389	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5390	* If it ever does, we want to know about it so log the exit code and bail.
5391	*/
5392	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5393	}
5394
5395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5396	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5397	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5398	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5399	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5400	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5401	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5402	#else
5403	case SVM_EXIT_CLGI:
5404	case SVM_EXIT_STGI:
5405	case SVM_EXIT_VMLOAD:
5406	case SVM_EXIT_VMSAVE:
5407	case SVM_EXIT_INVLPGA:
5408	case SVM_EXIT_VMRUN:
5409	#endif
5410	case SVM_EXIT_RSM:
5411	case SVM_EXIT_SKINIT:
5412	{
5413	hmR0SvmSetPendingXcptUD(pVCpu);
5414	return VINF_SUCCESS;
5415	}
5416
5417	case SVM_EXIT_XCPT_DE:
5418	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5419	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5420	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5421	case SVM_EXIT_XCPT_OF:
5422	case SVM_EXIT_XCPT_BR:
5423	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5424	case SVM_EXIT_XCPT_NM:
5425	case SVM_EXIT_XCPT_DF:
5426	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5427	case SVM_EXIT_XCPT_TS:
5428	case SVM_EXIT_XCPT_NP:
5429	case SVM_EXIT_XCPT_SS:
5430	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5431	/* SVM_EXIT_XCPT_PF: */
5432	case SVM_EXIT_XCPT_15: /* Reserved. */
5433	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5434	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5435	case SVM_EXIT_XCPT_MC:
5436	case SVM_EXIT_XCPT_XF:
5437	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5438	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5439	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5440	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5441
5442	case SVM_EXIT_SWINT:
5443	VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5444
5445	default:
5446	{
5447	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5448	pVCpu->hm.s.u32HMError = uExitCode;
5449	return VERR_SVM_UNKNOWN_EXIT;
5450	}
5451	}
5452	}
5453	}
5454	/* not reached */
5455	}
5456
5457
5458	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5459	*
5460	* The following few functions and associated structure contains the bloat
5461	* necessary for providing detailed debug events and dtrace probes as well as
5462	* reliable host side single stepping. This works on the principle of
5463	* "subclassing" the normal execution loop and workers. We replace the loop
5464	* method completely and override selected helpers to add necessary adjustments
5465	* to their core operation.
5466	*
5467	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5468	* any performance for debug and analysis features.
5469	*
5470	* @{
5471	*/
5472
5473	/**
5474	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5475	* the debug run loop.
5476	*/
5477	typedef struct SVMRUNDBGSTATE
5478	{
5479	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5480	uint64_t bmInterceptInitial;
5481	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5482	uint32_t bmXcptInitial;
5483	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5484	uint16_t bmInterceptRdCRxInitial;
5485	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5486	uint16_t bmInterceptWrCRxInitial;
5487	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5488	uint16_t bmInterceptRdDRxInitial;
5489	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5490	uint16_t bmInterceptWrDRxInitial;
5491
5492	/** Whether we've actually modified the intercept control qword. */
5493	bool fModifiedInterceptCtrl : 1;
5494	/** Whether we've actually modified the exception bitmap. */
5495	bool fModifiedXcptBitmap : 1;
5496	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5497	bool fModifiedInterceptRdCRx : 1;
5498	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5499	bool fModifiedInterceptWrCRx : 1;
5500	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5501	bool fModifiedInterceptRdDRx : 1;
5502	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5503	bool fModifiedInterceptWrDRx : 1;
5504
5505	/** The CS we started executing with. */
5506	uint16_t uCsStart;
5507	/** The RIP we started executing at. This is for detecting that we stepped. */
5508	uint64_t uRipStart;
5509
5510	/** The sequence number of the Dtrace provider settings the state was
5511	* configured against. */
5512	uint32_t uDtraceSettingsSeqNo;
5513	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5514	uint32_t bmXcptExtra;
5515	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5516	uint64_t bmInterceptExtra;
5517	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5518	uint16_t bmInterceptRdCRxExtra;
5519	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5520	uint16_t bmInterceptWrCRxExtra;
5521	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5522	uint16_t bmInterceptRdDRxExtra;
5523	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5524	uint16_t bmInterceptWrDRxExtra;
5525	/** VM-exits to check (one bit per VM-exit). */
5526	uint32_t bmExitsToCheck[33];
5527	} SVMRUNDBGSTATE;
5528	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5529	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5530
5531
5532	/**
5533	* Initializes the SVMRUNDBGSTATE structure.
5534	*
5535	* @param pVCpu The cross context virtual CPU structure of the
5536	* calling EMT.
5537	* @param pSvmTransient The SVM-transient structure.
5538	* @param pDbgState The debug state to initialize.
5539	*/
5540	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5541	{
5542	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5543	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5544	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5545	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5546	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5547	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5548	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5549
5550	pDbgState->fModifiedInterceptCtrl = false;
5551	pDbgState->fModifiedXcptBitmap = false;
5552	pDbgState->fModifiedInterceptRdCRx = false;
5553	pDbgState->fModifiedInterceptWrCRx = false;
5554	pDbgState->fModifiedInterceptRdDRx = false;
5555	pDbgState->fModifiedInterceptWrDRx = false;
5556
5557	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5558	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5559
5560	/* We don't really need to zero these. */
5561	pDbgState->bmInterceptExtra = 0;
5562	pDbgState->bmXcptExtra = 0;
5563	pDbgState->bmInterceptRdCRxExtra = 0;
5564	pDbgState->bmInterceptWrCRxExtra = 0;
5565	pDbgState->bmInterceptRdDRxExtra = 0;
5566	pDbgState->bmInterceptWrDRxExtra = 0;
5567	}
5568
5569
5570	/**
5571	* Updates the VMCB fields with changes requested by @a pDbgState.
5572	*
5573	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5574	* immediately before executing guest code, i.e. when interrupts are disabled.
5575	* We don't check status codes here as we cannot easily assert or return in the
5576	* latter case.
5577	*
5578	* @param pSvmTransient The SVM-transient structure.
5579	* @param pDbgState The debug state.
5580	*/
5581	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5582	{
5583	/*
5584	* Ensure desired flags in VMCS control fields are set.
5585	*/
5586	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5587	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5588	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5589	{ \
5590	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5591	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5592	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5593	(a_fModified) = true; \
5594	} \
5595	} while (0)
5596	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5597	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5598	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5599	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5600	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5601	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5602	#undef ADD_EXTRA_INTERCEPTS
5603	}
5604
5605
5606	/**
5607	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5608	* for re-entry next time around.
5609	*
5610	* @returns Strict VBox status code (i.e. informational status codes too).
5611	* @param pSvmTransient The SVM-transient structure.
5612	* @param pDbgState The debug state.
5613	*/
5614	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5615	{
5616	/*
5617	* Restore VM-exit control settings as we may not reenter this function the
5618	* next time around.
5619	*/
5620	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5621
5622	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5623	if ((a_fModified)) \
5624	{ \
5625	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5626	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5627	} \
5628	} while (0)
5629	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5630	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5631	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5632	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5633	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5634	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5635	#undef RESTORE_INTERCEPTS
5636	}
5637
5638
5639	/**
5640	* Configures VM-exit controls for current DBGF and DTrace settings.
5641	*
5642	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5643	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5644	*
5645	* @param pVCpu The cross context virtual CPU structure.
5646	* @param pSvmTransient The SVM-transient structure. May update
5647	* fUpdatedTscOffsettingAndPreemptTimer.
5648	* @param pDbgState The debug state.
5649	*/
5650	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5651	{
5652	/*
5653	* Take down the dtrace serial number so we can spot changes.
5654	*/
5655	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5656	ASMCompilerBarrier();
5657
5658	/*
5659	* Clear data members that we'll be rebuilding here.
5660	*/
5661	pDbgState->bmXcptExtra = 0;
5662	pDbgState->bmInterceptExtra = 0;
5663	pDbgState->bmInterceptRdCRxExtra = 0;
5664	pDbgState->bmInterceptWrCRxExtra = 0;
5665	pDbgState->bmInterceptRdDRxExtra = 0;
5666	pDbgState->bmInterceptWrDRxExtra = 0;
5667	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5668	pDbgState->bmExitsToCheck[i] = 0;
5669
5670	/*
5671	* Software interrupts (INT XXh)
5672	*/
5673	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5674	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5675	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5676	{
5677	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5678	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5679	}
5680
5681	/*
5682	* INT3 breakpoints - triggered by #BP exceptions.
5683	*/
5684	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5685	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5686
5687	/*
5688	* Exception bitmap and XCPT events+probes.
5689	*/
5690	#define SET_XCPT(a_iXcpt) do { \
5691	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5692	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5693	} while (0)
5694
5695	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5696	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5697	SET_XCPT(iXcpt);
5698
5699	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5700	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5701	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5702	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5703	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5704	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5705	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5706	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5707	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5708	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5709	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5710	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5711	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5712	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5713	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5714	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5715	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5716	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5717
5718	#undef SET_XCPT
5719
5720	/*
5721	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5722	*
5723	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5724	* So, when adding/changing/removing please don't forget to update it.
5725	*
5726	* Some of the macros are picking up local variables to save horizontal space,
5727	* (being able to see it in a table is the lesser evil here).
5728	*/
5729	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5730	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5731	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5732	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5733	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5734	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5735	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5736	} else do { } while (0)
5737	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5738	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5739	{ \
5740	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5741	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5742	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5743	} else do { } while (0)
5744
5745	/** @todo double check these */
5746	/** @todo Check what more AMD-V specific we can intercept. */
5747	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5748	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5749	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5750	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5751	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5752	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5753	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5754	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5755	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5756	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5757	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5759	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5761
5762	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5763	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5764	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5765	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5766	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5767	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5768	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5769	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5770	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5771	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5772	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5774	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5776	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5778
5779	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5780	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5781	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5782	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5783
5784	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5785	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5786	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5787	{
5788	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5789	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5790	}
5791
5792	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5793	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5794	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5795	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5796
5797	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5798	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5799	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5800	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5801
5802	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5804	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5806	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5808	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5809	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5810	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5811	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5812	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5813	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5814	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5816	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5818	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5819	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5820	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5821	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5822	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5823	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5824	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5825	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5826	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5827	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5828	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5829	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5830	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5832	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5833	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5834
5835	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5836	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5837
5838	#undef IS_EITHER_ENABLED
5839	#undef SET_ONLY_XBM_IF_EITHER_EN
5840	#undef SET_INCP_XBM_IF_EITHER_EN
5841
5842	/*
5843	* Sanitize the control stuff.
5844	*/
5845	/** @todo filter out unsupported stuff? */
5846	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5847	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5848	{
5849	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5850	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5851	RT_NOREF(pSvmTransient);
5852	}
5853
5854	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5855	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5856	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5857	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5858	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5859	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5860	pDbgState->bmExitsToCheck[0],
5861	pDbgState->bmExitsToCheck[1],
5862	pDbgState->bmExitsToCheck[2],
5863	pDbgState->bmExitsToCheck[3],
5864	pDbgState->bmExitsToCheck[4]));
5865	}
5866
5867
5868	/**
5869	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5870	* appropriate.
5871	*
5872	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5873	* bitmap.
5874	*
5875	* @returns Strict VBox status code (i.e. informational status codes too).
5876	* @param pVCpu The cross context virtual CPU structure.
5877	* @param pSvmTransient The SVM-transient structure.
5878	* @param uExitCode The VM-exit code.
5879	*
5880	* @remarks The name of this function is displayed by dtrace, so keep it short
5881	* and to the point. No longer than 33 chars long, please.
5882	*/
5883	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5884	{
5885	/*
5886	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5887	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5888	*
5889	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5890	* does. Must add/change/remove both places. Same ordering, please.
5891	*
5892	* Added/removed events must also be reflected in the next section
5893	* where we dispatch dtrace events.
5894	*/
5895	bool fDtrace1 = false;
5896	bool fDtrace2 = false;
5897	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5898	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5899	uint64_t uEventArg = 0;
5900	#define SET_XCPT(a_XcptName) \
5901	do { \
5902	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5903	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5904	} while (0)
5905	#define SET_EXIT(a_EventSubName) \
5906	do { \
5907	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5908	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5909	} while (0)
5910	#define SET_BOTH(a_EventSubName) \
5911	do { \
5912	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5913	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5914	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5915	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5916	} while (0)
5917	switch (uExitCode)
5918	{
5919	case SVM_EXIT_SWINT:
5920	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5921	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5922	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5923	break;
5924
5925	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5926	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5927	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5928	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5929	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5930	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5931	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5932	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5933	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5934	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5935	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5936	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5937	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5938	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5939	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5940	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5941	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5942	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5943
5944	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5945	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5946	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5947	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5948	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5949	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5950	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5951	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5952	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5953	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5954	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5955	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5956	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5957	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5958
5959	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5960	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5961	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5962	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5963	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5964	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5965	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5966	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5967	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5968	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5969	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5970	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5971	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5972	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5973	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5974
5975	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5976	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5977	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5978	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5979	SET_BOTH(CRX_READ);
5980	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5981	break;
5982	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5983	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5984	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5985	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
5986	case SVM_EXIT_CR0_SEL_WRITE:
5987	SET_BOTH(CRX_WRITE);
5988	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
5989	break;
5990	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5991	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
5992	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
5993	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5994	SET_BOTH(DRX_READ);
5995	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
5996	break;
5997	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5998	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
5999	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6000	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6001	SET_BOTH(DRX_WRITE);
6002	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6003	break;
6004	case SVM_EXIT_MSR:
6005	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6006	SET_BOTH(WRMSR);
6007	else
6008	SET_BOTH(RDMSR);
6009	break;
6010	case SVM_EXIT_MWAIT_ARMED:
6011	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6012	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6013	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6014	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6015	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6016	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6017	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6018	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6019	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6020	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6021	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6022	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6023	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6024
6025	case SVM_EXIT_SHUTDOWN:
6026	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6027	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6028	break;
6029
6030	default:
6031	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6032	break;
6033	}
6034	#undef SET_BOTH
6035	#undef SET_EXIT
6036
6037	/*
6038	* Dtrace tracepoints go first. We do them here at once so we don't
6039	* have to copy the guest state saving and stuff a few dozen times.
6040	* Down side is that we've got to repeat the switch, though this time
6041	* we use enmEvent since the probes are a subset of what DBGF does.
6042	*/
6043	if (fDtrace1 \|\| fDtrace2)
6044	{
6045	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6046	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6047	switch (enmEvent1)
6048	{
6049	/** @todo consider which extra parameters would be helpful for each probe. */
6050	case DBGFEVENT_END: break;
6051	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6052	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6053	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6054	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6055	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6056	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6057	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6058	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6059	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6060	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6061	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6062	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6063	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6064	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6065	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6066	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6067	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6068	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6069	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6070	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6071	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6072	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6073	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6074	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6075	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6076	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6077	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6078	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6079	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6080	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6081	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6082	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6083	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6084	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6085	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6086	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6087	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6088	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6089	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6090	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6091	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6092	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6093	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6094	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6095	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6096	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6097	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6098	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6099	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6100	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6101	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6102	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6103	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6104	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6105	}
6106	switch (enmEvent2)
6107	{
6108	/** @todo consider which extra parameters would be helpful for each probe. */
6109	case DBGFEVENT_END: break;
6110	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6111	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6112	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6113	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6114	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6115	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6116	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6117	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6118	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6119	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6120	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6121	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6122	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6123	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6124	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6125	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6126	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6127	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6128	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6129	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6130	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6131	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6132	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6133	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6134	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6135	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6136	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6137	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6138	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6139	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6140	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6141	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6142	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6144	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6145	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6146	}
6147	}
6148
6149	/*
6150	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6151	* the DBGF call will do a full check).
6152	*
6153	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6154	* Note! If we have to events, we prioritize the first, i.e. the instruction
6155	* one, in order to avoid event nesting.
6156	*/
6157	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6158	VBOXSTRICTRC rcStrict;
6159	if ( enmEvent1 != DBGFEVENT_END
6160	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6161	{
6162	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6163	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6164	}
6165	else if ( enmEvent2 != DBGFEVENT_END
6166	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6167	{
6168	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6169	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6170	}
6171	else
6172	rcStrict = VINF_SUCCESS;
6173	return rcStrict;
6174	}
6175
6176
6177	/**
6178	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6179	* debug variant.
6180	*
6181	* @returns Strict VBox status code (informational status codes included).
6182	* @param pVCpu The cross context virtual CPU structure.
6183	* @param pSvmTransient Pointer to the SVM transient structure.
6184	* @param pDbgState The runtime debug state.
6185	*/
6186	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6187	{
6188	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6189	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6190
6191	/*
6192	* Expensive (saves context) generic dtrace VM-exit probe.
6193	*/
6194	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6195	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6196	{ /* more likely */ }
6197	else
6198	{
6199	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6200	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6201	}
6202
6203	/*
6204	* Check for single stepping event if we're stepping.
6205	*/
6206	if (pVCpu->hm.s.fSingleInstruction)
6207	{
6208	switch (uExitCode)
6209	{
6210	/* Various events: */
6211	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6212	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6213	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6214	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6215	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6216	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6217	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6218	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6219	case SVM_EXIT_INTR:
6220	case SVM_EXIT_NMI:
6221	case SVM_EXIT_VINTR:
6222	case SVM_EXIT_NPF:
6223	case SVM_EXIT_AVIC_NOACCEL:
6224
6225	/* Instruction specific VM-exits: */
6226	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6227	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6228	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6229	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6230	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6231	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6232	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6233	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6234	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6235	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6236	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6237	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6238	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6239	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6240	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6241	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6242	case SVM_EXIT_CR0_SEL_WRITE:
6243	case SVM_EXIT_IDTR_READ:
6244	case SVM_EXIT_GDTR_READ:
6245	case SVM_EXIT_LDTR_READ:
6246	case SVM_EXIT_TR_READ:
6247	case SVM_EXIT_IDTR_WRITE:
6248	case SVM_EXIT_GDTR_WRITE:
6249	case SVM_EXIT_LDTR_WRITE:
6250	case SVM_EXIT_TR_WRITE:
6251	case SVM_EXIT_RDTSC:
6252	case SVM_EXIT_RDPMC:
6253	case SVM_EXIT_PUSHF:
6254	case SVM_EXIT_POPF:
6255	case SVM_EXIT_CPUID:
6256	case SVM_EXIT_RSM:
6257	case SVM_EXIT_IRET:
6258	case SVM_EXIT_SWINT:
6259	case SVM_EXIT_INVD:
6260	case SVM_EXIT_PAUSE:
6261	case SVM_EXIT_HLT:
6262	case SVM_EXIT_INVLPG:
6263	case SVM_EXIT_INVLPGA:
6264	case SVM_EXIT_IOIO:
6265	case SVM_EXIT_MSR:
6266	case SVM_EXIT_TASK_SWITCH:
6267	case SVM_EXIT_VMRUN:
6268	case SVM_EXIT_VMMCALL:
6269	case SVM_EXIT_VMLOAD:
6270	case SVM_EXIT_VMSAVE:
6271	case SVM_EXIT_STGI:
6272	case SVM_EXIT_CLGI:
6273	case SVM_EXIT_SKINIT:
6274	case SVM_EXIT_RDTSCP:
6275	case SVM_EXIT_ICEBP:
6276	case SVM_EXIT_WBINVD:
6277	case SVM_EXIT_MONITOR:
6278	case SVM_EXIT_MWAIT:
6279	case SVM_EXIT_MWAIT_ARMED:
6280	case SVM_EXIT_XSETBV:
6281	case SVM_EXIT_RDPRU:
6282	case SVM_EXIT_WRITE_EFER_TRAP:
6283	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6284	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6285	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6286	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6287	case SVM_EXIT_MCOMMIT:
6288	{
6289	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6290	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6291	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6292	{
6293	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6294	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6295	return VINF_EM_DBG_STEPPED;
6296	}
6297	break;
6298	}
6299
6300	/* Errors and unexpected events: */
6301	case SVM_EXIT_FERR_FREEZE:
6302	case SVM_EXIT_SHUTDOWN:
6303	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6304	break;
6305
6306	case SVM_EXIT_SMI:
6307	case SVM_EXIT_INIT:
6308	default:
6309	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6310	break;
6311	}
6312	}
6313
6314	/*
6315	* Check for debugger event breakpoints and dtrace probes.
6316	*/
6317	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6318	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6319	{
6320	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6321	if (rcStrict != VINF_SUCCESS)
6322	{
6323	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6324	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6325	return rcStrict;
6326	}
6327	}
6328
6329	/*
6330	* Normal processing.
6331	*/
6332	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6333	}
6334
6335
6336	/**
6337	* Runs the guest code using AMD-V in single step mode.
6338	*
6339	* @returns Strict VBox status code.
6340	* @param pVCpu The cross context virtual CPU structure.
6341	* @param pcLoops Pointer to the number of executed loops.
6342	*/
6343	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6344	{
6345	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6346	Assert(pcLoops);
6347	Assert(*pcLoops <= cMaxResumeLoops);
6348
6349	SVMTRANSIENT SvmTransient;
6350	RT_ZERO(SvmTransient);
6351	SvmTransient.fUpdateTscOffsetting = true;
6352	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6353
6354	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6355
6356	/* Set HMCPU indicators. */
6357	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6358	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6359	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6360	pVCpu->hmr0.s.fUsingDebugLoop = true;
6361
6362	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6363	SVMRUNDBGSTATE DbgState;
6364	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6365	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6366
6367	/*
6368	* The loop.
6369	*/
6370	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6371	for (;;)
6372	{
6373	Assert(!HMR0SuspendPending());
6374	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6375	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6376	(unsigned)RTMpCpuId(), *pcLoops));
6377	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6378
6379	/* Set up VM-execution controls the next two can respond to. */
6380	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6381
6382	/* Preparatory work for running nested-guest code, this may force us to return to
6383	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6384	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6385	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6386	if (rc != VINF_SUCCESS)
6387	break;
6388
6389	/*
6390	* No longjmps to ring-3 from this point on!!!
6391	*
6392	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6393	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6394	*/
6395	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6396
6397	/* Override any obnoxious code in the above two calls. */
6398	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6399
6400	/*
6401	* Finally execute guest code.
6402	*/
6403	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6404
6405	/* Restore any residual host-state and save any bits shared between host and guest
6406	into the guest-CPU state. Re-enables interrupts! */
6407	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6408
6409	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6410	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6411	{ /* very likely*/ }
6412	else
6413	{
6414	if (rc == VINF_SUCCESS)
6415	rc = VERR_SVM_INVALID_GUEST_STATE;
6416	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6417	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6418	return rc;
6419	}
6420
6421	/* Handle the #VMEXIT. */
6422	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6423	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6424	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6425	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6426	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6427	if (rc != VINF_SUCCESS)
6428	break;
6429	if (++(*pcLoops) >= cMaxResumeLoops)
6430	{
6431	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6432	rc = VINF_EM_RAW_INTERRUPT;
6433	break;
6434	}
6435
6436	/*
6437	* Stepping: Did the RIP change, if so, consider it a single step.
6438	* Otherwise, make sure one of the TFs gets set.
6439	*/
6440	if (fStepping)
6441	{
6442	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6443	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6444	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6445	{
6446	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6447	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6448	rc = VINF_EM_DBG_STEPPED;
6449	break;
6450	}
6451	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6452	}
6453
6454	/*
6455	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6456	* Revert the state changes afterware so we can drop intercepts no longer needed.
6457	*/
6458	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6459	{
6460	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6461	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6462	}
6463	}
6464
6465	/*
6466	* Clear the X86_EFL_TF if necessary.
6467	*/
6468	if (pVCpu->hmr0.s.fClearTrapFlag)
6469	{
6470	pVCpu->hmr0.s.fClearTrapFlag = false;
6471	pCtx->eflags.Bits.u1TF = 0;
6472	}
6473
6474	/* Restore HMCPU indicators. */
6475	pVCpu->hmr0.s.fUsingDebugLoop = false;
6476	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6477	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6478
6479	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6480	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6481
6482	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6483	return rc;
6484	}
6485
6486	/** @} */
6487
6488	#undef VMEXIT_CALL_RET
6489
6490
6491	#ifdef VBOX_STRICT
6492	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6493	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6494	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6495
6496	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6497	do \
6498	{ \
6499	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6500	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6501	} while (0)
6502
6503	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6504	do { \
6505	AssertPtr((a_pVCpu)); \
6506	AssertPtr((a_pSvmTransient)); \
6507	Assert(ASMIntAreEnabled()); \
6508	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6509	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6510	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6511	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6512	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6513	HMSVM_ASSERT_PREEMPT_CPUID(); \
6514	} while (0)
6515	#else
6516	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6517	do { \
6518	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6519	} while (0)
6520	#endif
6521
6522
6523	/**
6524	* Gets the IEM exception flags for the specified SVM event.
6525	*
6526	* @returns The IEM exception flags.
6527	* @param pEvent Pointer to the SVM event.
6528	*
6529	* @remarks This function currently only constructs flags required for
6530	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6531	* and CR2 aspects of an exception are not included).
6532	*/
6533	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6534	{
6535	uint8_t const uEventType = pEvent->n.u3Type;
6536	uint32_t fIemXcptFlags;
6537	switch (uEventType)
6538	{
6539	case SVM_EVENT_EXCEPTION:
6540	/*
6541	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6542	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6543	*/
6544	if (pEvent->n.u8Vector == X86_XCPT_BP)
6545	{
6546	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6547	break;
6548	}
6549	if (pEvent->n.u8Vector == X86_XCPT_OF)
6550	{
6551	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6552	break;
6553	}
6554	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6555	RT_FALL_THRU();
6556	case SVM_EVENT_NMI:
6557	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6558	break;
6559
6560	case SVM_EVENT_EXTERNAL_IRQ:
6561	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6562	break;
6563
6564	case SVM_EVENT_SOFTWARE_INT:
6565	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6566	break;
6567
6568	default:
6569	fIemXcptFlags = 0;
6570	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6571	break;
6572	}
6573	return fIemXcptFlags;
6574	}
6575
6576
6577	/**
6578	* Handle a condition that occurred while delivering an event through the guest
6579	* IDT.
6580	*
6581	* @returns VBox status code (informational error codes included).
6582	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6583	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6584	* continue execution of the guest which will delivery the \#DF.
6585	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6586	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6587	*
6588	* @param pVCpu The cross context virtual CPU structure.
6589	* @param pSvmTransient Pointer to the SVM transient structure.
6590	*
6591	* @remarks No-long-jump zone!!!
6592	*/
6593	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6594	{
6595	int rc = VINF_SUCCESS;
6596	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6597	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6598
6599	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6600	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6601	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6602
6603	/*
6604	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6605	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6606	*
6607	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6608	*/
6609	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6610	{
6611	IEMXCPTRAISE enmRaise;
6612	IEMXCPTRAISEINFO fRaiseInfo;
6613	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6614	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6615	if (fExitIsHwXcpt)
6616	{
6617	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6618	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6619	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6620	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6621	}
6622	else
6623	{
6624	/*
6625	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6626	* then we end up here.
6627	*
6628	* If the event was:
6629	* - a software interrupt, we can re-execute the instruction which will
6630	* regenerate the event.
6631	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6632	* - a hardware exception or external interrupt, we re-inject it.
6633	*/
6634	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6635	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6636	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6637	else
6638	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6639	}
6640
6641	switch (enmRaise)
6642	{
6643	case IEMXCPTRAISE_CURRENT_XCPT:
6644	case IEMXCPTRAISE_PREV_EVENT:
6645	{
6646	/* For software interrupts, we shall re-execute the instruction. */
6647	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6648	{
6649	RTGCUINTPTR GCPtrFaultAddress = 0;
6650
6651	/* If we are re-injecting an NMI, clear NMI blocking. */
6652	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6653	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6654
6655	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6656	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6657	{
6658	pSvmTransient->fVectoringPF = true;
6659	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6660	pVCpu->cpum.GstCtx.cr2));
6661	}
6662	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6663	&& uIdtVector == X86_XCPT_PF)
6664	{
6665	/*
6666	* If the previous exception was a #PF, we need to recover the CR2 value.
6667	* This can't happen with shadow paging.
6668	*/
6669	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6670	}
6671
6672	/*
6673	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6674	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6675	*/
6676	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6677	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6678	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6679
6680	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6681	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6682	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6683	}
6684	break;
6685	}
6686
6687	case IEMXCPTRAISE_REEXEC_INSTR:
6688	{
6689	Assert(rc == VINF_SUCCESS);
6690	break;
6691	}
6692
6693	case IEMXCPTRAISE_DOUBLE_FAULT:
6694	{
6695	/*
6696	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6697	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6698	* converted into a #DF.
6699	*/
6700	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6701	{
6702	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6703	pSvmTransient->fVectoringDoublePF = true;
6704	Assert(rc == VINF_SUCCESS);
6705	}
6706	else
6707	{
6708	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6709	hmR0SvmSetPendingXcptDF(pVCpu);
6710	rc = VINF_HM_DOUBLE_FAULT;
6711	}
6712	break;
6713	}
6714
6715	case IEMXCPTRAISE_TRIPLE_FAULT:
6716	{
6717	rc = VINF_EM_RESET;
6718	break;
6719	}
6720
6721	case IEMXCPTRAISE_CPU_HANG:
6722	{
6723	rc = VERR_EM_GUEST_CPU_HANG;
6724	break;
6725	}
6726
6727	default:
6728	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6729	}
6730	}
6731	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6732	return rc;
6733	}
6734
6735
6736	/**
6737	* Advances the guest RIP by the number of bytes specified in @a cb.
6738	*
6739	* @param pVCpu The cross context virtual CPU structure.
6740	* @param cb RIP increment value in bytes.
6741	*/
6742	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6743	{
6744	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6745	pCtx->rip += cb;
6746
6747	/* Update interrupt shadow. */
6748	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
6749	&& pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
6750	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6751	}
6752
6753
6754	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6755	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6756	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6757
6758	/** @name \#VMEXIT handlers.
6759	* @{
6760	*/
6761
6762	/**
6763	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6764	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6765	*/
6766	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6767	{
6768	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6769
6770	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6771	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6772	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6773	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6774
6775	/*
6776	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6777	* signal -before- the timer fires if the current interrupt is our own timer or a some
6778	* other host interrupt. We also cannot examine what interrupt it is until the host
6779	* actually take the interrupt.
6780	*
6781	* Going back to executing guest code here unconditionally causes random scheduling
6782	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6783	*/
6784	return VINF_EM_RAW_INTERRUPT;
6785	}
6786
6787
6788	/**
6789	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6790	*/
6791	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6792	{
6793	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6794
6795	VBOXSTRICTRC rcStrict;
6796	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6797	if (fSupportsNextRipSave)
6798	{
6799	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6800	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6801	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6802	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6803	}
6804	else
6805	{
6806	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6807	rcStrict = IEMExecOne(pVCpu);
6808	}
6809
6810	if (rcStrict == VINF_IEM_RAISED_XCPT)
6811	{
6812	rcStrict = VINF_SUCCESS;
6813	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6814	}
6815	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6816	return rcStrict;
6817	}
6818
6819
6820	/**
6821	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6822	*/
6823	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6824	{
6825	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6826
6827	VBOXSTRICTRC rcStrict;
6828	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6829	if (fSupportsNextRipSave)
6830	{
6831	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6832	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6833	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6834	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6835	}
6836	else
6837	{
6838	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6839	rcStrict = IEMExecOne(pVCpu);
6840	}
6841
6842	if (rcStrict == VINF_IEM_RAISED_XCPT)
6843	{
6844	rcStrict = VINF_SUCCESS;
6845	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6846	}
6847	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6848	return rcStrict;
6849	}
6850
6851
6852	/**
6853	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6854	*/
6855	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6856	{
6857	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6858
6859	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6860	VBOXSTRICTRC rcStrict;
6861	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6862	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6863	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6864	if (!pExitRec)
6865	{
6866	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6867	if (fSupportsNextRipSave)
6868	{
6869	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6870	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6871	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6872	}
6873	else
6874	{
6875	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6876	rcStrict = IEMExecOne(pVCpu);
6877	}
6878
6879	if (rcStrict == VINF_IEM_RAISED_XCPT)
6880	{
6881	rcStrict = VINF_SUCCESS;
6882	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6883	}
6884	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6885	}
6886	else
6887	{
6888	/*
6889	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6890	*/
6891	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6892
6893	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6894	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6895
6896	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6897
6898	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6899	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6900	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6901	}
6902	return rcStrict;
6903	}
6904
6905
6906	/**
6907	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6908	*/
6909	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6910	{
6911	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6912
6913	VBOXSTRICTRC rcStrict;
6914	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6915	if (fSupportsNextRipSave)
6916	{
6917	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6918	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6919	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6920	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6921	}
6922	else
6923	{
6924	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6925	rcStrict = IEMExecOne(pVCpu);
6926	}
6927
6928	if (rcStrict == VINF_SUCCESS)
6929	pSvmTransient->fUpdateTscOffsetting = true;
6930	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6931	{
6932	rcStrict = VINF_SUCCESS;
6933	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6934	}
6935	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6936	return rcStrict;
6937	}
6938
6939
6940	/**
6941	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6942	*/
6943	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6944	{
6945	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6946
6947	VBOXSTRICTRC rcStrict;
6948	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6949	if (fSupportsNextRipSave)
6950	{
6951	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6952	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6953	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6954	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6955	}
6956	else
6957	{
6958	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6959	rcStrict = IEMExecOne(pVCpu);
6960	}
6961
6962	if (rcStrict == VINF_SUCCESS)
6963	pSvmTransient->fUpdateTscOffsetting = true;
6964	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6965	{
6966	rcStrict = VINF_SUCCESS;
6967	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6968	}
6969	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6970	return rcStrict;
6971	}
6972
6973
6974	/**
6975	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6976	*/
6977	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6978	{
6979	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6980
6981	VBOXSTRICTRC rcStrict;
6982	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6983	if (fSupportsNextRipSave)
6984	{
6985	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6986	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6987	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6988	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
6989	}
6990	else
6991	{
6992	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6993	rcStrict = IEMExecOne(pVCpu);
6994	}
6995
6996	if (rcStrict == VINF_IEM_RAISED_XCPT)
6997	{
6998	rcStrict = VINF_SUCCESS;
6999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7000	}
7001	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7002	return rcStrict;
7003	}
7004
7005
7006	/**
7007	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7008	*/
7009	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7010	{
7011	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7012	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7013
7014	VBOXSTRICTRC rcStrict;
7015	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7016	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7017	if ( fSupportsDecodeAssists
7018	&& fSupportsNextRipSave)
7019	{
7020	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7021	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7022	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7023	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7024	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7025	}
7026	else
7027	{
7028	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7029	rcStrict = IEMExecOne(pVCpu);
7030	}
7031
7032	if (rcStrict == VINF_IEM_RAISED_XCPT)
7033	{
7034	rcStrict = VINF_SUCCESS;
7035	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7036	}
7037	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7038	return VBOXSTRICTRC_VAL(rcStrict);
7039	}
7040
7041
7042	/**
7043	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7044	*/
7045	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7046	{
7047	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7048
7049	VBOXSTRICTRC rcStrict;
7050	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7051	if (fSupportsNextRipSave)
7052	{
7053	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7054	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7055	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7056	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7057	}
7058	else
7059	{
7060	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7061	rcStrict = IEMExecOne(pVCpu);
7062	}
7063
7064	if ( rcStrict == VINF_EM_HALT
7065	\|\| rcStrict == VINF_SUCCESS)
7066	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7067	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7068	{
7069	rcStrict = VINF_SUCCESS;
7070	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7071	}
7072	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7073	if (rcStrict != VINF_SUCCESS)
7074	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7075	return VBOXSTRICTRC_VAL(rcStrict);;
7076	}
7077
7078
7079	/**
7080	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7081	*/
7082	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7083	{
7084	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7085
7086	/*
7087	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7088	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7089	* segment override prefix or other prefixes might be used, in which case we fallback to
7090	* IEMExecOne() to figure out.
7091	*/
7092	VBOXSTRICTRC rcStrict;
7093	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7094	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7095	if (cbInstr)
7096	{
7097	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7098	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7099	}
7100	else
7101	{
7102	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7103	rcStrict = IEMExecOne(pVCpu);
7104	}
7105
7106	if (rcStrict == VINF_IEM_RAISED_XCPT)
7107	{
7108	rcStrict = VINF_SUCCESS;
7109	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7110	}
7111	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7112	return rcStrict;
7113	}
7114
7115
7116	/**
7117	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7118	*/
7119	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7120	{
7121	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7122
7123	VBOXSTRICTRC rcStrict;
7124	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7125	if (fSupportsNextRipSave)
7126	{
7127	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7128	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7129	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7130	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7131	}
7132	else
7133	{
7134	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7135	rcStrict = IEMExecOne(pVCpu);
7136	}
7137
7138	if ( rcStrict == VINF_EM_HALT
7139	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7140	rcStrict = VINF_SUCCESS;
7141	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7142	{
7143	rcStrict = VINF_SUCCESS;
7144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7145	}
7146	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7147	return rcStrict;
7148	}
7149
7150
7151	/**
7152	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7153	* \#VMEXIT.
7154	*/
7155	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7156	{
7157	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7158	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7159	return VINF_EM_RESET;
7160	}
7161
7162
7163	/**
7164	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7165	*/
7166	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7167	{
7168	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7169	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7170	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7171	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7172	RT_NOREF(pVmcb);
7173	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7174	return VERR_SVM_UNEXPECTED_EXIT;
7175	}
7176
7177
7178	/**
7179	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7180	*/
7181	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7182	{
7183	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7184
7185	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7186	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
7187	#ifdef VBOX_WITH_STATISTICS
7188	switch (pSvmTransient->u64ExitCode)
7189	{
7190	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7191	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7192	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7193	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7194	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7195	}
7196	#endif
7197
7198	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7199	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7200	if ( fSupportsDecodeAssists
7201	&& fSupportsNextRipSave)
7202	{
7203	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7204	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7205	if (fMovCRx)
7206	{
7207	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7208	\| CPUMCTX_EXTRN_APIC_TPR);
7209	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7210	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7211	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7212	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7213	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7214	return VBOXSTRICTRC_VAL(rcStrict);
7215	}
7216	/* else: SMSW instruction, fall back below to IEM for this. */
7217	}
7218
7219	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7220	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7221	AssertMsg( rcStrict == VINF_SUCCESS
7222	\|\| rcStrict == VINF_PGM_SYNC_CR3
7223	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7224	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7225	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7226	if (rcStrict == VINF_IEM_RAISED_XCPT)
7227	{
7228	rcStrict = VINF_SUCCESS;
7229	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7230	}
7231	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7232	return rcStrict;
7233	}
7234
7235
7236	/**
7237	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7238	*/
7239	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7240	{
7241	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7242
7243	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7244	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7245	Assert(iCrReg <= 15);
7246
7247	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7248	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7249	bool fDecodedInstr = false;
7250	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7251	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7252	if ( fSupportsDecodeAssists
7253	&& fSupportsNextRipSave)
7254	{
7255	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7256	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7257	if (fMovCRx)
7258	{
7259	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7260	\| CPUMCTX_EXTRN_APIC_TPR);
7261	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7262	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7263	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7264	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7265	fDecodedInstr = true;
7266	}
7267	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7268	}
7269
7270	if (!fDecodedInstr)
7271	{
7272	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7273	Log4Func(("iCrReg=%#x\n", iCrReg));
7274	rcStrict = IEMExecOne(pVCpu);
7275	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7276	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7277	rcStrict = VERR_EM_INTERPRETER;
7278	}
7279
7280	if (rcStrict == VINF_SUCCESS)
7281	{
7282	switch (iCrReg)
7283	{
7284	case 0:
7285	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7286	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7287	break;
7288
7289	case 2:
7290	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7292	break;
7293
7294	case 3:
7295	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7296	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7297	break;
7298
7299	case 4:
7300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7302	break;
7303
7304	case 8:
7305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7307	break;
7308
7309	default:
7310	{
7311	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7312	pSvmTransient->u64ExitCode, iCrReg));
7313	break;
7314	}
7315	}
7316	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7317	}
7318	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7319	{
7320	rcStrict = VINF_SUCCESS;
7321	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7322	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7323	}
7324	else
7325	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7326	return rcStrict;
7327	}
7328
7329
7330	/**
7331	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7332	*
7333	* @returns Strict VBox status code.
7334	* @param pVCpu The cross context virtual CPU structure.
7335	* @param pVmcb Pointer to the VM control block.
7336	*/
7337	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7338	{
7339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7340	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7341
7342	VBOXSTRICTRC rcStrict;
7343	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7344	if (fSupportsNextRipSave)
7345	{
7346	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7347	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7348	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7349	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7350	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7351	}
7352	else
7353	{
7354	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7355	rcStrict = IEMExecOne(pVCpu);
7356	}
7357
7358	AssertMsg( rcStrict == VINF_SUCCESS
7359	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7360	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7361	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7362
7363	if (rcStrict == VINF_IEM_RAISED_XCPT)
7364	{
7365	rcStrict = VINF_SUCCESS;
7366	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7367	}
7368	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7369	return rcStrict;
7370	}
7371
7372
7373	/**
7374	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7375	*
7376	* @returns Strict VBox status code.
7377	* @param pVCpu The cross context virtual CPU structure.
7378	* @param pVmcb Pointer to the VM control block.
7379	* @param pSvmTransient Pointer to the SVM-transient structure.
7380	*/
7381	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7382	{
7383	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7384	uint32_t const idMsr = pCtx->ecx;
7385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7386	Log4Func(("idMsr=%#RX32\n", idMsr));
7387
7388	/*
7389	* Handle TPR patching MSR writes.
7390	* We utilitize the LSTAR MSR for patching.
7391	*/
7392	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7393	if ( idMsr == MSR_K8_LSTAR
7394	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7395	{
7396	unsigned cbInstr;
7397	if (fSupportsNextRipSave)
7398	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7399	else
7400	{
7401	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7402	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7403	if ( rc == VINF_SUCCESS
7404	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7405	Assert(cbInstr > 0);
7406	else
7407	cbInstr = 0;
7408	}
7409
7410	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7411	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7412	{
7413	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7414	AssertRCReturn(rc, rc);
7415	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7416	}
7417
7418	int rc = VINF_SUCCESS;
7419	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7420	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7421	return rc;
7422	}
7423
7424	/*
7425	* Handle regular MSR writes.
7426	*/
7427	VBOXSTRICTRC rcStrict;
7428	if (fSupportsNextRipSave)
7429	{
7430	/** @todo Optimize this: We don't need to get much of the MSR state here
7431	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7432	* clear the applicable extern flags. */
7433	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7434	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7435	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7436	}
7437	else
7438	{
7439	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7440	rcStrict = IEMExecOne(pVCpu);
7441	}
7442
7443	AssertMsg( rcStrict == VINF_SUCCESS
7444	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7445	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7446	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7447
7448	if (rcStrict == VINF_SUCCESS)
7449	{
7450	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7451	if ( idMsr >= MSR_IA32_X2APIC_START
7452	&& idMsr <= MSR_IA32_X2APIC_END)
7453	{
7454	/*
7455	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7456	* When full APIC register virtualization is implemented we'll have to make sure
7457	* APIC state is saved from the VMCB before IEM changes it.
7458	*/
7459	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7460	}
7461	else
7462	{
7463	switch (idMsr)
7464	{
7465	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7466	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7467	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7468	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7469	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7470	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7471	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7472	}
7473	}
7474	}
7475	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7476	{
7477	rcStrict = VINF_SUCCESS;
7478	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7479	}
7480	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7481	return rcStrict;
7482	}
7483
7484
7485	/**
7486	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7487	* \#VMEXIT.
7488	*/
7489	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7490	{
7491	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7492
7493	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7494	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7495	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7496
7497	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7498	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7499	}
7500
7501
7502	/**
7503	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7504	*/
7505	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7506	{
7507	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7508	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7509
7510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7511
7512	/** @todo Stepping with nested-guest. */
7513	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7514	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7515	{
7516	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7517	if (pSvmTransient->fWasGuestDebugStateActive)
7518	{
7519	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7520	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7521	return VERR_SVM_UNEXPECTED_EXIT;
7522	}
7523
7524	/*
7525	* Lazy DR0-3 loading.
7526	*/
7527	if (!pSvmTransient->fWasHyperDebugStateActive)
7528	{
7529	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7530	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7531
7532	/* Don't intercept DRx read and writes. */
7533	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7534	pVmcb->ctrl.u16InterceptRdDRx = 0;
7535	pVmcb->ctrl.u16InterceptWrDRx = 0;
7536	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7537
7538	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7539	VMMRZCallRing3Disable(pVCpu);
7540	HM_DISABLE_PREEMPT(pVCpu);
7541
7542	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7543	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7544	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7545
7546	HM_RESTORE_PREEMPT();
7547	VMMRZCallRing3Enable(pVCpu);
7548
7549	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7550	return VINF_SUCCESS;
7551	}
7552	}
7553
7554	/*
7555	* Interpret the read/writing of DRx.
7556	*/
7557	/** @todo Decode assist. */
7558	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
7559	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7560	if (RT_LIKELY(rc == VINF_SUCCESS))
7561	{
7562	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7563	/** @todo CPUM should set this flag! */
7564	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7565	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7566	}
7567	else
7568	Assert(rc == VERR_EM_INTERPRETER);
7569	return rc;
7570	}
7571
7572
7573	/**
7574	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7575	*/
7576	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7577	{
7578	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7579	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7580	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7581	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7582	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7583	return rc;
7584	}
7585
7586
7587	/**
7588	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7589	*/
7590	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7591	{
7592	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7593	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7594
7595	/** @todo decode assists... */
7596	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7597	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7598	{
7599	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7600	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7601	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7602	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7603	{
7604	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7605	hmR0SvmUpdateVmRunFunction(pVCpu);
7606	}
7607	}
7608	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7609	{
7610	rcStrict = VINF_SUCCESS;
7611	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7612	}
7613	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7614	return rcStrict;
7615	}
7616
7617
7618	/**
7619	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7620	*/
7621	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7622	{
7623	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7624	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7625
7626	/* I/O operation lookup arrays. */
7627	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7628	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7629	the result (in AL/AX/EAX). */
7630	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7631	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7632	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7633
7634	Log4Func(("CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
7635
7636	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7637	SVMIOIOEXITINFO IoExitInfo;
7638	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7639	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7640	uint32_t cbValue = s_aIOSize[uIOWidth];
7641	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7642
7643	if (RT_UNLIKELY(!cbValue))
7644	{
7645	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7646	return VERR_EM_INTERPRETER;
7647	}
7648
7649	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7650	VBOXSTRICTRC rcStrict;
7651	PCEMEXITREC pExitRec = NULL;
7652	if ( !pVCpu->hm.s.fSingleInstruction
7653	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7654	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7655	!IoExitInfo.n.u1Str
7656	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7657	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7658	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7659	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7660	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7661	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7662	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7663	if (!pExitRec)
7664	{
7665	bool fUpdateRipAlready = false;
7666	if (IoExitInfo.n.u1Str)
7667	{
7668	/* INS/OUTS - I/O String instruction. */
7669	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7670	* in EXITINFO1? Investigate once this thing is up and running. */
7671	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7672	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7673	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7674	static IEMMODE const s_aenmAddrMode[8] =
7675	{
7676	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7677	};
7678	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7679	if (enmAddrMode != (IEMMODE)-1)
7680	{
7681	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7682	if (cbInstr <= 15 && cbInstr >= 1)
7683	{
7684	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7685	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7686	{
7687	/* Don't know exactly how to detect whether u3Seg is valid, currently
7688	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7689	2384 Opterons when only checking NRIP. */
7690	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7691	if ( fSupportsNextRipSave
7692	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7693	{
7694	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7695	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7696	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7697	IoExitInfo.n.u3Seg, true /fIoChecked/);
7698	}
7699	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7700	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7701	X86_SREG_DS, true /fIoChecked/);
7702	else
7703	rcStrict = IEMExecOne(pVCpu);
7704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7705	}
7706	else
7707	{
7708	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7709	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7710	true /fIoChecked/);
7711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7712	}
7713	}
7714	else
7715	{
7716	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7717	rcStrict = IEMExecOne(pVCpu);
7718	}
7719	}
7720	else
7721	{
7722	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7723	rcStrict = IEMExecOne(pVCpu);
7724	}
7725	fUpdateRipAlready = true;
7726	}
7727	else
7728	{
7729	/* IN/OUT - I/O instruction. */
7730	Assert(!IoExitInfo.n.u1Rep);
7731
7732	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7733	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7734	{
7735	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7736	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7737	&& !pCtx->eflags.Bits.u1TF)
7738	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7739	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7740	}
7741	else
7742	{
7743	uint32_t u32Val = 0;
7744	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7745	if (IOM_SUCCESS(rcStrict))
7746	{
7747	/* Save result of I/O IN instr. in AL/AX/EAX. */
7748	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7749	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7750	}
7751	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7752	&& !pCtx->eflags.Bits.u1TF)
7753	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7754
7755	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7756	}
7757	}
7758
7759	if (IOM_SUCCESS(rcStrict))
7760	{
7761	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7762	if (!fUpdateRipAlready)
7763	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7764
7765	/*
7766	* If any I/O breakpoints are armed, we need to check if one triggered
7767	* and take appropriate action.
7768	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7769	*/
7770	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7771	* execution engines about whether hyper BPs and such are pending. */
7772	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7773	uint32_t const uDr7 = pCtx->dr[7];
7774	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7775	&& X86_DR7_ANY_RW_IO(uDr7)
7776	&& (pCtx->cr4 & X86_CR4_DE))
7777	\|\| DBGFBpIsHwIoArmed(pVM)))
7778	{
7779	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7780	VMMRZCallRing3Disable(pVCpu);
7781	HM_DISABLE_PREEMPT(pVCpu);
7782
7783	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7784	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7785
7786	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7787	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7788	{
7789	/* Raise #DB. */
7790	pVmcb->guest.u64DR6 = pCtx->dr[6];
7791	pVmcb->guest.u64DR7 = pCtx->dr[7];
7792	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7793	hmR0SvmSetPendingXcptDB(pVCpu);
7794	}
7795	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7796	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7797	else if ( rcStrict2 != VINF_SUCCESS
7798	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7799	rcStrict = rcStrict2;
7800	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7801
7802	HM_RESTORE_PREEMPT();
7803	VMMRZCallRing3Enable(pVCpu);
7804	}
7805
7806	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7807	}
7808
7809	#ifdef VBOX_STRICT
7810	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7811	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7812	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7813	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7814	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7815	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7816	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7817	else
7818	{
7819	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7820	* statuses, that the VMM device and some others may return. See
7821	* IOM_SUCCESS() for guidance. */
7822	AssertMsg( RT_FAILURE(rcStrict)
7823	\|\| rcStrict == VINF_SUCCESS
7824	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7825	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7826	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7827	\|\| rcStrict == VINF_EM_DBG_STEPPED
7828	\|\| rcStrict == VINF_EM_RAW_TO_R3
7829	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7830	}
7831	#endif
7832	}
7833	else
7834	{
7835	/*
7836	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7837	*/
7838	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7839	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7840	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7841	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7842	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7843	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7844	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7845
7846	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7847	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7848
7849	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7850	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7851	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7852	}
7853	return rcStrict;
7854	}
7855
7856
7857	/**
7858	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7859	*/
7860	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7861	{
7862	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7863	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7864	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7865
7866	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7867	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7868	Assert(pVM->hmr0.s.fNestedPaging);
7869
7870	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7871	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7872	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7873	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7874	thus intentionally not copied into u32ErrCode. */
7875
7876	Log4Func(("#NPF at CS:RIP=%04x:%#RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7877	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7878
7879	/*
7880	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7881	*/
7882	if ( pVM->hm.s.fTprPatchingAllowed
7883	&& (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7884	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7885	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7886	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7887	&& !CPUMIsGuestInLongModeEx(pCtx)
7888	&& !CPUMGetGuestCPL(pVCpu)
7889	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7890	{
7891	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7892	GCPhysApicBase &= PAGE_BASE_GC_MASK;
7893
7894	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7895	{
7896	/* Only attempt to patch the instruction once. */
7897	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7898	if (!pPatch)
7899	return VINF_EM_HM_PATCH_TPR_INSTR;
7900	}
7901	}
7902
7903	/*
7904	* Determine the nested paging mode.
7905	*/
7906	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7907	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7908
7909	/*
7910	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7911	*/
7912	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7913	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7914	{
7915	/*
7916	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7917	* injecting the original pending event would most likely cause the same MMIO #NPF.
7918	*/
7919	if (pVCpu->hm.s.Event.fPending)
7920	{
7921	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7922	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7923	}
7924
7925	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7926	VBOXSTRICTRC rcStrict;
7927	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7928	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7929	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7930	if (!pExitRec)
7931	{
7932
7933	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7934	u32ErrCode);
7935
7936	/*
7937	* If we succeed, resume guest execution.
7938	*
7939	* If we fail in interpreting the instruction because we couldn't get the guest
7940	* physical address of the page containing the instruction via the guest's page
7941	* tables (we would invalidate the guest page in the host TLB), resume execution
7942	* which would cause a guest page fault to let the guest handle this weird case.
7943	*
7944	* See @bugref{6043}.
7945	*/
7946	if ( rcStrict == VINF_SUCCESS
7947	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7948	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7949	{
7950	/* Successfully handled MMIO operation. */
7951	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7952	rcStrict = VINF_SUCCESS;
7953	}
7954	}
7955	else
7956	{
7957	/*
7958	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7959	*/
7960	Assert(pCtx == &pVCpu->cpum.GstCtx);
7961	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7962	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7963	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7964
7965	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7966	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7967
7968	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7969	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7970	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7971	}
7972	return rcStrict;
7973	}
7974
7975	/*
7976	* Nested page-fault.
7977	*/
7978	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
7979	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
7980	TRPMResetTrap(pVCpu);
7981
7982	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
7983
7984	/*
7985	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
7986	*/
7987	if ( rc == VINF_SUCCESS
7988	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7989	\|\| rc == VERR_PAGE_NOT_PRESENT)
7990	{
7991	/* We've successfully synced our shadow page tables. */
7992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7993	rc = VINF_SUCCESS;
7994	}
7995
7996	/*
7997	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
7998	* re-inject the original event.
7999	*/
8000	if (pVCpu->hm.s.Event.fPending)
8001	{
8002	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8003
8004	/*
8005	* If the #NPF handler requested emulation of the instruction, ignore it.
8006	* We need to re-inject the original event so as to not lose it.
8007	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8008	* LiveCD is broken for other reasons).
8009	*/
8010	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8011	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8012	}
8013
8014	return rc;
8015	}
8016
8017
8018	/**
8019	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8020	* \#VMEXIT.
8021	*/
8022	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8023	{
8024	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8025	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8026
8027	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8028	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8029	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8030
8031	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8032	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8033	return VINF_SUCCESS;
8034	}
8035
8036
8037	/**
8038	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8039	* \#VMEXIT.
8040	*/
8041	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8042	{
8043	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8044	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8045
8046	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8047	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8048	#endif
8049
8050	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8051	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8052	{
8053	/*
8054	* AMD-V provides us with the exception which caused the TS; we collect
8055	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8056	*/
8057	Log4Func(("TS occurred during event delivery\n"));
8058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8059	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8060	}
8061
8062	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8063	* emulation. */
8064	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8065	return VERR_EM_INTERPRETER;
8066	}
8067
8068
8069	/**
8070	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8071	*/
8072	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8073	{
8074	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8076
8077	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8078	if (pVM->hm.s.fTprPatchingAllowed)
8079	{
8080	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8081	if (rc != VERR_NOT_FOUND)
8082	{
8083	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8084	return rc;
8085	}
8086	}
8087
8088	if (EMAreHypercallInstructionsEnabled(pVCpu))
8089	{
8090	unsigned cbInstr;
8091	if (hmR0SvmSupportsNextRipSave(pVCpu))
8092	{
8093	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8094	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8095	}
8096	else
8097	{
8098	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8099	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8100	if ( rc == VINF_SUCCESS
8101	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8102	Assert(cbInstr > 0);
8103	else
8104	cbInstr = 0;
8105	}
8106
8107	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8108	if (RT_SUCCESS(rcStrict))
8109	{
8110	/* Only update the RIP if we're continuing guest execution and not in the case
8111	of say VINF_GIM_R3_HYPERCALL. */
8112	if (rcStrict == VINF_SUCCESS)
8113	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8114
8115	return VBOXSTRICTRC_VAL(rcStrict);
8116	}
8117	else
8118	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8119	}
8120
8121	hmR0SvmSetPendingXcptUD(pVCpu);
8122	return VINF_SUCCESS;
8123	}
8124
8125
8126	/**
8127	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8128	*/
8129	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8130	{
8131	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8132
8133	unsigned cbInstr;
8134	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8135	if (fSupportsNextRipSave)
8136	{
8137	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8138	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8139	}
8140	else
8141	{
8142	PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8143	int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8144	if ( rc == VINF_SUCCESS
8145	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8146	Assert(cbInstr > 0);
8147	else
8148	cbInstr = 0;
8149	}
8150
8151	/** @todo The guest has likely hit a contended spinlock. We might want to
8152	* poke a schedule different guest VCPU. */
8153	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8154	return VINF_EM_RAW_INTERRUPT;
8155	}
8156
8157
8158	/**
8159	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8160	* \#VMEXIT.
8161	*/
8162	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8163	{
8164	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8165	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8166	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8167
8168	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8169	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8170	}
8171
8172
8173	/**
8174	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8175	*/
8176	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8177	{
8178	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8179
8180	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8181	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8182	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8183
8184	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8185	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8186	* handler executes. Yes, x86 is ugly.
8187	*/
8188	return VINF_EM_RAW_EMULATE_INSTR;
8189	}
8190
8191
8192	/**
8193	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8194	* Conditional \#VMEXIT.
8195	*/
8196	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8197	{
8198	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8199	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8200	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8201
8202	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8203	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8204	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8205	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8206	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8207	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8208
8209	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8210	if (pVM->hmr0.s.fNestedPaging)
8211	{
8212	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8213	if ( !pSvmTransient->fVectoringDoublePF
8214	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8215	{
8216	/* A genuine guest #PF, reflect it to the guest. */
8217	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8218	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8219	uFaultAddress, uErrCode));
8220	}
8221	else
8222	{
8223	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8224	hmR0SvmSetPendingXcptDF(pVCpu);
8225	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8226	}
8227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8228	return VINF_SUCCESS;
8229	}
8230	#endif
8231
8232	Assert(!pVM->hmr0.s.fNestedPaging);
8233
8234	/*
8235	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8236	*/
8237	if ( pVM->hm.s.fTprPatchingAllowed
8238	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8239	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8240	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8241	&& !CPUMIsGuestInLongModeEx(pCtx)
8242	&& !CPUMGetGuestCPL(pVCpu)
8243	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8244	{
8245	RTGCPHYS GCPhysApicBase;
8246	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8247	GCPhysApicBase &= PAGE_BASE_GC_MASK;
8248
8249	/* Check if the page at the fault-address is the APIC base. */
8250	PGMPTWALK Walk;
8251	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8252	if ( rc2 == VINF_SUCCESS
8253	&& Walk.GCPhys == GCPhysApicBase)
8254	{
8255	/* Only attempt to patch the instruction once. */
8256	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8257	if (!pPatch)
8258	return VINF_EM_HM_PATCH_TPR_INSTR;
8259	}
8260	}
8261
8262	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8263	pCtx->rip, uErrCode, pCtx->cr3));
8264
8265	/*
8266	* If it's a vectoring #PF, emulate injecting the original event injection as
8267	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8268	* event injection that caused a #PF. See @bugref{6607}.
8269	*/
8270	if (pSvmTransient->fVectoringPF)
8271	{
8272	Assert(pVCpu->hm.s.Event.fPending);
8273	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8274	}
8275
8276	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8277	int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
8278
8279	Log4Func(("#PF: rc=%Rrc\n", rc));
8280
8281	if (rc == VINF_SUCCESS)
8282	{
8283	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8284	TRPMResetTrap(pVCpu);
8285	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8287	return rc;
8288	}
8289
8290	if (rc == VINF_EM_RAW_GUEST_TRAP)
8291	{
8292	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8293
8294	/*
8295	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8296	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8297	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8298	*/
8299	if ( !pSvmTransient->fVectoringDoublePF
8300	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8301	{
8302	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8303	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8304	TRPMResetTrap(pVCpu);
8305
8306	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8307	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8308	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8309	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8310	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8311	#endif
8312
8313	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8314	}
8315	else
8316	{
8317	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8318	TRPMResetTrap(pVCpu);
8319	hmR0SvmSetPendingXcptDF(pVCpu);
8320	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8321	}
8322
8323	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8324	return VINF_SUCCESS;
8325	}
8326
8327	TRPMResetTrap(pVCpu);
8328	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8329	return rc;
8330	}
8331
8332
8333	/**
8334	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8335	* Conditional \#VMEXIT.
8336	*/
8337	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8338	{
8339	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8340	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8341	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8342
8343	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8344	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8345	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8346
8347	/** @todo if we accumulate more optional stuff here, we ought to combine the
8348	* reading of opcode bytes to avoid doing more than once. */
8349
8350	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8351	if (pVCpu->hm.s.fGIMTrapXcptUD)
8352	{
8353	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8354	uint8_t cbInstr = 0;
8355	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8356	if (rcStrict == VINF_SUCCESS)
8357	{
8358	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8359	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8360	rcStrict = VINF_SUCCESS;
8361	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8362	}
8363	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8364	rcStrict = VINF_SUCCESS;
8365	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8366	rcStrict = VINF_GIM_R3_HYPERCALL;
8367	else
8368	{
8369	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8370	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8371	}
8372	}
8373
8374	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8375	{
8376	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8377	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8378	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8379	{
8380	/* Ideally, IEM should just handle all these special #UD situations, but
8381	we don't quite trust things to behave optimially when doing that. So,
8382	for now we'll restrict ourselves to a handful of possible sysenter and
8383	sysexit encodings that we filter right here. */
8384	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8385	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8386	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8387	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8388	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8389	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8390	{
8391	cbInstr = cbMin;
8392	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8393	AssertRCStmt(rc2, cbInstr = 0);
8394	}
8395	else
8396	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8397	if ( cbInstr == 0 /* read error */
8398	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8399	\|\| ( uCpl == 0
8400	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8401	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8402	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8403	{
8404	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8405	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8406	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8407	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPtrInstr, abInstr, cbInstr);
8408	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8409	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8410	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8413	if (rcStrict == VINF_IEM_RAISED_XCPT)
8414	rcStrict = VINF_SUCCESS;
8415	return rcStrict;
8416	}
8417	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8418	}
8419	else
8420	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8421	}
8422
8423	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8424	if (RT_FAILURE(rcStrict))
8425	{
8426	hmR0SvmSetPendingXcptUD(pVCpu);
8427	rcStrict = VINF_SUCCESS;
8428	}
8429
8430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8431	return rcStrict;
8432	}
8433
8434
8435	/**
8436	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8437	* Conditional \#VMEXIT.
8438	*/
8439	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8440	{
8441	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8442	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8443	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8444
8445	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8446	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8447
8448	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8449	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8450
8451	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8452
8453	if (!(pCtx->cr0 & X86_CR0_NE))
8454	{
8455	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8456	PDISSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8457	unsigned cbInstr;
8458	int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
8459	if (RT_SUCCESS(rc))
8460	{
8461	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8462	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8463	if (RT_SUCCESS(rc))
8464	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8465	}
8466	else
8467	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8468	return rc;
8469	}
8470
8471	hmR0SvmSetPendingXcptMF(pVCpu);
8472	return VINF_SUCCESS;
8473	}
8474
8475
8476	/**
8477	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8478	* \#VMEXIT.
8479	*/
8480	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8481	{
8482	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8483	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8484	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8486
8487	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8488	{
8489	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8490	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8491	}
8492
8493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8494
8495	/*
8496	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8497	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8498	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8499	*/
8500	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8501	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8502	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8503	int rc = DBGFTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8504	if (rc == VINF_EM_RAW_GUEST_TRAP)
8505	{
8506	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8507	if (CPUMIsHyperDebugStateActive(pVCpu))
8508	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8509
8510	/* Reflect the exception back to the guest. */
8511	hmR0SvmSetPendingXcptDB(pVCpu);
8512	rc = VINF_SUCCESS;
8513	}
8514
8515	/*
8516	* Update DR6.
8517	*/
8518	if (CPUMIsHyperDebugStateActive(pVCpu))
8519	{
8520	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8521	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8522	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8523	}
8524	else
8525	{
8526	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8527	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8528	}
8529
8530	return rc;
8531	}
8532
8533
8534	/**
8535	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8536	* Conditional \#VMEXIT.
8537	*/
8538	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8539	{
8540	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8541	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8542	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8543
8544	SVMEVENT Event;
8545	Event.u = 0;
8546	Event.n.u1Valid = 1;
8547	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8548	Event.n.u8Vector = X86_XCPT_AC;
8549	Event.n.u1ErrorCodeValid = 1;
8550	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8551	return VINF_SUCCESS;
8552	}
8553
8554
8555	/**
8556	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8557	* Conditional \#VMEXIT.
8558	*/
8559	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8560	{
8561	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8562	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8563	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8565
8566	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8567	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
8568	if (rc == VINF_EM_RAW_GUEST_TRAP)
8569	{
8570	SVMEVENT Event;
8571	Event.u = 0;
8572	Event.n.u1Valid = 1;
8573	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8574	Event.n.u8Vector = X86_XCPT_BP;
8575	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8576	rc = VINF_SUCCESS;
8577	}
8578
8579	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8580	return rc;
8581	}
8582
8583
8584	/**
8585	* Hacks its way around the lovely mesa driver's backdoor accesses.
8586	*
8587	* @sa hmR0VmxHandleMesaDrvGp
8588	*/
8589	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8590	{
8591	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8592	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8593	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8594	RT_NOREF(pCtx, pVmcb);
8595
8596	/* For now we'll just skip the instruction. */
8597	hmR0SvmAdvanceRip(pVCpu, 1);
8598	return VINF_SUCCESS;
8599	}
8600
8601
8602	/**
8603	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8604	* backdoor logging w/o checking what it is running inside.
8605	*
8606	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8607	* backdoor port and magic numbers loaded in registers.
8608	*
8609	* @returns true if it is, false if it isn't.
8610	* @sa hmR0VmxIsMesaDrvGp
8611	*/
8612	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8613	{
8614	/* Check magic and port. */
8615	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8616	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8617	if (pCtx->dx != UINT32_C(0x5658))
8618	return false;
8619	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8620	return false;
8621
8622	/* Check that it is #GP(0). */
8623	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8624	return false;
8625
8626	/* Flat ring-3 CS. */
8627	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8628	if (pVmcb->guest.u8CPL != 3)
8629	return false;
8630	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8631	return false;
8632
8633	/* 0xed: IN eAX,dx */
8634	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8635	{
8636	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8637	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8638	uint8_t abInstr[1];
8639	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8640	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8641	if (RT_FAILURE(rc))
8642	return false;
8643	if (abInstr[0] != 0xed)
8644	return false;
8645	}
8646	else
8647	{
8648	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8649	if (pVmcb->ctrl.abInstr[0] != 0xed)
8650	return false;
8651	}
8652	return true;
8653	}
8654
8655
8656	/**
8657	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8658	* Conditional \#VMEXIT.
8659	*/
8660	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8661	{
8662	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8663	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8665
8666	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8667	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8668
8669	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8670	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8671	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8672	{
8673	SVMEVENT Event;
8674	Event.u = 0;
8675	Event.n.u1Valid = 1;
8676	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8677	Event.n.u8Vector = X86_XCPT_GP;
8678	Event.n.u1ErrorCodeValid = 1;
8679	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8680	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8681	return VINF_SUCCESS;
8682	}
8683	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8684	}
8685
8686
8687	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(VBOX_WITH_NESTED_HWVIRT_SVM)
8688	/**
8689	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8690	*/
8691	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8692	{
8693	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8694	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8695
8696	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8697	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8698	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8699	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8700	Assert(uVector <= X86_XCPT_LAST);
8701	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8702
8703	SVMEVENT Event;
8704	Event.u = 0;
8705	Event.n.u1Valid = 1;
8706	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8707	Event.n.u8Vector = uVector;
8708	switch (uVector)
8709	{
8710	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8711	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8712	case X86_XCPT_DF:
8713	case X86_XCPT_TS:
8714	case X86_XCPT_NP:
8715	case X86_XCPT_SS:
8716	case X86_XCPT_GP:
8717	case X86_XCPT_AC:
8718	{
8719	Event.n.u1ErrorCodeValid = 1;
8720	Event.n.u32ErrorCode = uErrCode;
8721	break;
8722	}
8723	}
8724
8725	#ifdef VBOX_WITH_STATISTICS
8726	switch (uVector)
8727	{
8728	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8729	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8730	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8731	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8732	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8733	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8734	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8735	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8736	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8737	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8738	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8739	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8740	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8741	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8742	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8743	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8744	default:
8745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8746	break;
8747	}
8748	#endif
8749
8750	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8751	return VINF_SUCCESS;
8752	}
8753	#endif
8754
8755	/**
8756	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8757	*/
8758	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8759	{
8760	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8761	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8762
8763	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8764	SVMEVENT Event;
8765	Event.u = 0;
8766	Event.n.u1Valid = 1;
8767	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8768	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8769	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8770	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8771	return VINF_SUCCESS;
8772	}
8773
8774
8775	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8776	/**
8777	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8778	*/
8779	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8780	{
8781	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8782
8783	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8784	Assert(pVmcb);
8785	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8786
8787	VBOXSTRICTRC rcStrict;
8788	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8789	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8790	if (fSupportsNextRipSave)
8791	{
8792	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8793	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8794	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8795	}
8796	else
8797	{
8798	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8799	rcStrict = IEMExecOne(pVCpu);
8800	}
8801
8802	if (rcStrict == VINF_SUCCESS)
8803	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8804	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8805	{
8806	rcStrict = VINF_SUCCESS;
8807	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8808	}
8809	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8810	return rcStrict;
8811	}
8812
8813
8814	/**
8815	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8816	*/
8817	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8818	{
8819	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8820
8821	/*
8822	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8823	* we only intercept STGI when events are pending for GIF to become 1.
8824	*/
8825	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8826	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8827	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8828
8829	VBOXSTRICTRC rcStrict;
8830	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8831	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8832	if (fSupportsNextRipSave)
8833	{
8834	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8835	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8836	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8837	}
8838	else
8839	{
8840	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8841	rcStrict = IEMExecOne(pVCpu);
8842	}
8843
8844	if (rcStrict == VINF_SUCCESS)
8845	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8846	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8847	{
8848	rcStrict = VINF_SUCCESS;
8849	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8850	}
8851	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8852	return rcStrict;
8853	}
8854
8855
8856	/**
8857	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8858	*/
8859	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8860	{
8861	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8862
8863	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8864	Assert(pVmcb);
8865	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8866
8867	VBOXSTRICTRC rcStrict;
8868	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8869	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
8870	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
8871	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
8872	if (fSupportsNextRipSave)
8873	{
8874	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8875	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8876	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
8877	}
8878	else
8879	{
8880	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8881	rcStrict = IEMExecOne(pVCpu);
8882	}
8883
8884	if (rcStrict == VINF_SUCCESS)
8885	{
8886	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
8887	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
8888	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
8889	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
8890	}
8891	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8892	{
8893	rcStrict = VINF_SUCCESS;
8894	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8895	}
8896	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8897	return rcStrict;
8898	}
8899
8900
8901	/**
8902	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
8903	*/
8904	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8905	{
8906	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8907
8908	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8909	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8910
8911	VBOXSTRICTRC rcStrict;
8912	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8913	if (fSupportsNextRipSave)
8914	{
8915	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
8916	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8917	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
8918	}
8919	else
8920	{
8921	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
8922	rcStrict = IEMExecOne(pVCpu);
8923	}
8924
8925	if (rcStrict == VINF_IEM_RAISED_XCPT)
8926	{
8927	rcStrict = VINF_SUCCESS;
8928	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8929	}
8930	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8931	return rcStrict;
8932	}
8933
8934
8935	/**
8936	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
8937	*/
8938	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8939	{
8940	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8941
8942	VBOXSTRICTRC rcStrict;
8943	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8944	if (fSupportsNextRipSave)
8945	{
8946	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
8947	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8948	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8949	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
8950	}
8951	else
8952	{
8953	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
8954	rcStrict = IEMExecOne(pVCpu);
8955	}
8956
8957	if (rcStrict == VINF_IEM_RAISED_XCPT)
8958	{
8959	rcStrict = VINF_SUCCESS;
8960	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8961	}
8962	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8963	return rcStrict;
8964	}
8965
8966
8967	/**
8968	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
8969	*/
8970	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8971	{
8972	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8973	/* We shall import the entire state here, just in case we enter and continue execution of
8974	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
8975	could lose lose part of CPU state. */
8976	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8977
8978	VBOXSTRICTRC rcStrict;
8979	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8980	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
8981	if (fSupportsNextRipSave)
8982	{
8983	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8984	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8985	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
8986	}
8987	else
8988	{
8989	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
8990	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
8991	instruction itself, see @bugref{7243#c126} */
8992	rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
8993	}
8994	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
8995
8996	if (rcStrict == VINF_SUCCESS)
8997	{
8998	rcStrict = VINF_SVM_VMRUN;
8999	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9000	}
9001	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9002	{
9003	rcStrict = VINF_SUCCESS;
9004	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9005	}
9006	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9007	return rcStrict;
9008	}
9009
9010
9011	/**
9012	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9013	* Unconditional \#VMEXIT.
9014	*/
9015	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9016	{
9017	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9018	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9019
9020	if (pVCpu->hm.s.Event.fPending)
9021	{
9022	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9023	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9024	}
9025
9026	hmR0SvmSetPendingXcptDB(pVCpu);
9027	return VINF_SUCCESS;
9028	}
9029
9030
9031	/**
9032	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9033	* Conditional \#VMEXIT.
9034	*/
9035	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9036	{
9037	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9038	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9039
9040	SVMEVENT Event;
9041	Event.u = 0;
9042	Event.n.u1Valid = 1;
9043	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9044	Event.n.u8Vector = X86_XCPT_BP;
9045	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9046	return VINF_SUCCESS;
9047	}
9048	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9049
9050	/** @} */
9051

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 93100

Download in other formats: