HMSVMR0.cpp@ 107861

Last change on this file since 107861 was 107854, checked in by vboxsync, 4 weeks ago
x86.h,VMM: More AMD CPUID bits; addressed some old todos related to these; fixed bugs in svn & vmx world switcher (sanity checks, ++). jiraref:VBP-947 bugref:10738
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 381.5 KB

Line
1	/* $Id: HMSVMR0.cpp 107854 2025-01-18 23:59:26Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/pdmapic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT,
609	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
610	if (RT_FAILURE(rc))
611	return rc;
612
613	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
614	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
615
616	/* Set all bits to intercept all IO accesses. */
617	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
618
619	return VINF_SUCCESS;
620	}
621
622
623	/**
624	* Does global AMD-V termination (called during module termination).
625	*/
626	VMMR0DECL(void) SVMR0GlobalTerm(void)
627	{
628	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
629	{
630	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
631	g_pvIOBitmap = NULL;
632	g_HCPhysIOBitmap = 0;
633	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
634	}
635	}
636
637
638	/**
639	* Frees any allocated per-VCPU structures for a VM.
640	*
641	* @param pVM The cross context VM structure.
642	*/
643	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
644	{
645	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
646	{
647	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
648	AssertPtr(pVCpu);
649
650	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
651	{
652	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
653	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
654	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
655	}
656
657	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
658	{
659	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
660	pVCpu->hmr0.s.svm.pVmcb = NULL;
661	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
662	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
663	}
664
665	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
666	{
667	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
668	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
669	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
670	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
671	}
672	}
673	}
674
675
676	/**
677	* Sets pfnVMRun to the best suited variant.
678	*
679	* This must be called whenever anything changes relative to the SVMR0VMRun
680	* variant selection:
681	* - pVCpu->hm.s.fLoadSaveGuestXcr0
682	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
684	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
685	* - Perhaps: CPUMCTX.fXStateMask (windows only)
686	*
687	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
688	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
689	*/
690	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
691	{
692	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
693	{
694	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit_SansSpecCtrl },
695	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit_SansSpecCtrl },
696	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit_SansSpecCtrl },
697	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit_SansSpecCtrl },
698	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit_SansSpecCtrl },
699	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit_SansSpecCtrl },
700	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit_SansSpecCtrl },
701	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit_SansSpecCtrl },
702	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit_WithSpecCtrl },
703	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit_WithSpecCtrl },
704	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit_WithSpecCtrl },
705	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit_WithSpecCtrl },
706	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit_WithSpecCtrl },
707	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit_WithSpecCtrl },
708	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit_WithSpecCtrl },
709	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit_WithSpecCtrl },
710	};
711	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
712	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
713	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0)
714	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_SPEC_CTRL ? 8 : 0);
715	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
716	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
717	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
718	}
719
720
721	/**
722	* Selector FNHMSVMVMRUN implementation.
723	*/
724	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
725	{
726	hmR0SvmUpdateVmRunFunction(pVCpu);
727	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
728	}
729
730
731	/**
732	* Does per-VM AMD-V initialization.
733	*
734	* @returns VBox status code.
735	* @param pVM The cross context VM structure.
736	*/
737	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
738	{
739	int rc = VERR_INTERNAL_ERROR_5;
740
741	/*
742	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
743	*/
744	uint32_t u32Family;
745	uint32_t u32Model;
746	uint32_t u32Stepping;
747	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
748	{
749	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
750	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
751	}
752
753	/*
754	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
755	*/
756	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
757	{
758	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
759	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
760	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
761	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
762	}
763
764	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
765	{
766	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
767
768	/*
769	* Initialize the hardware-assisted SVM guest-execution handler.
770	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
771	*/
772	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
773
774	/*
775	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
776	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
777	*/
778	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
779	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
780	if (RT_FAILURE(rc))
781	goto failure_cleanup;
782
783	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
784	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
785	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
786
787	/*
788	* Allocate one page for the guest-state VMCB.
789	*/
790	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT,
791	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
792	if (RT_FAILURE(rc))
793	goto failure_cleanup;
794
795	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
796	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
797	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
798
799	/*
800	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
801	* SVM to not require one.
802	*/
803	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
804	NIL_RTHCPHYS /PhysHighest/, false /* fExecutable */);
805	if (RT_FAILURE(rc))
806	goto failure_cleanup;
807
808	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
809	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
810	/* Set all bits to intercept all MSR accesses (changed later on). */
811	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
812	}
813
814	return VINF_SUCCESS;
815
816	failure_cleanup:
817	hmR0SvmFreeStructs(pVM);
818	return rc;
819	}
820
821
822	/**
823	* Does per-VM AMD-V termination.
824	*
825	* @returns VBox status code.
826	* @param pVM The cross context VM structure.
827	*/
828	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
829	{
830	hmR0SvmFreeStructs(pVM);
831	return VINF_SUCCESS;
832	}
833
834
835	/**
836	* Returns whether the VMCB Clean Bits feature is supported.
837	*
838	* @returns @c true if supported, @c false otherwise.
839	* @param pVCpu The cross context virtual CPU structure.
840	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
841	*/
842	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
843	{
844	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
845	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
846	if (!fIsNestedGuest)
847	return fHostVmcbCleanBits;
848	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
849	}
850
851
852	/**
853	* Returns whether the decode assists feature is supported.
854	*
855	* @returns @c true if supported, @c false otherwise.
856	* @param pVCpu The cross context virtual CPU structure.
857	*/
858	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
859	{
860	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
861	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
862	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
863	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
864	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
865	#endif
866	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
867	}
868
869
870	/**
871	* Returns whether the NRIP_SAVE feature is supported.
872	*
873	* @returns @c true if supported, @c false otherwise.
874	* @param pVCpu The cross context virtual CPU structure.
875	*/
876	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
877	{
878	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
879	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
880	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
881	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
882	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
883	#endif
884	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
885	}
886
887
888	/**
889	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
890	*
891	* @param pVCpu The cross context virtual CPU structure.
892	* @param pbMsrBitmap Pointer to the MSR bitmap.
893	* @param idMsr The MSR for which the permissions are being set.
894	* @param enmRead MSR read permissions.
895	* @param enmWrite MSR write permissions.
896	*
897	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
898	* caller needs to take care of this.
899	*/
900	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
901	SVMMSREXITWRITE enmWrite)
902	{
903	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
904	uint16_t offMsrpm;
905	uint8_t uMsrpmBit;
906	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
907	AssertRC(rc);
908
909	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
910	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
911
912	pbMsrBitmap += offMsrpm;
913	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
914	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
915	else
916	{
917	if (!fInNestedGuestMode)
918	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
919	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
920	else
921	{
922	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
923	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
924	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
925	else
926	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
927	}
928	#endif
929	}
930
931	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
932	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
933	else
934	{
935	if (!fInNestedGuestMode)
936	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
937	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
938	else
939	{
940	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
941	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
942	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
943	else
944	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
945	}
946	#endif
947	}
948	}
949
950
951	/**
952	* Sets up AMD-V for the specified VM.
953	* This function is only called once per-VM during initalization.
954	*
955	* @returns VBox status code.
956	* @param pVM The cross context VM structure.
957	*/
958	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
959	{
960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
961	AssertReturn(pVM, VERR_INVALID_PARAMETER);
962
963	/*
964	* Validate and copy over some parameters.
965	*/
966	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
967	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
968	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
969	pVM->hmr0.s.fNestedPaging = fNestedPaging;
970	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
971
972	/*
973	* Determin some configuration parameters.
974	*/
975	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
976	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
977	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
978
979	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
980	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
981
982	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
983	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
984	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
985
986	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
987	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
988	#endif
989
990	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
991	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
992	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
993	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
994
995	/* Always trap #AC for reasons of security. */
996	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
997
998	/* Always trap #DB for reasons of security. */
999	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
1000
1001	/* Trap exceptions unconditionally (debug purposes). */
1002	#ifdef HMSVM_ALWAYS_TRAP_PF
1003	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
1004	#endif
1005	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1006	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1007	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1008	\| RT_BIT_32(X86_XCPT_DE)
1009	\| RT_BIT_32(X86_XCPT_NM)
1010	\| RT_BIT_32(X86_XCPT_UD)
1011	\| RT_BIT_32(X86_XCPT_NP)
1012	\| RT_BIT_32(X86_XCPT_SS)
1013	\| RT_BIT_32(X86_XCPT_GP)
1014	\| RT_BIT_32(X86_XCPT_PF)
1015	\| RT_BIT_32(X86_XCPT_MF)
1016	;
1017	#endif
1018
1019	/* Apply the exceptions intercepts needed by the GIM provider. */
1020	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1021	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1022
1023	/* Apply the exceptions intercepts needed by the GCM fixers. */
1024	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1025	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1026
1027	/* The mesa 3d driver hack needs #GP. */
1028	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1029	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1030
1031	/* Set up unconditional intercepts and conditions. */
1032	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1033	\| SVM_CTRL_INTERCEPT_VMMCALL
1034	\| SVM_CTRL_INTERCEPT_VMSAVE
1035	\| SVM_CTRL_INTERCEPT_VMLOAD
1036	\| SVM_CTRL_INTERCEPT_CLGI
1037	\| SVM_CTRL_INTERCEPT_STGI;
1038
1039	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1040	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1041	#endif
1042
1043	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1044	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1045	{
1046	/* Virtualized VMSAVE/VMLOAD. */
1047	if (fUseVirtVmsaveVmload)
1048	{
1049	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1050	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1051	\| SVM_CTRL_INTERCEPT_VMLOAD);
1052	}
1053	else
1054	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1055
1056	/* Virtual GIF. */
1057	if (fUseVGif)
1058	{
1059	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1060	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1061	\| SVM_CTRL_INTERCEPT_STGI);
1062	}
1063	else
1064	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1065	}
1066	else
1067	#endif
1068	{
1069	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1070	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1071	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1072	}
1073
1074	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1075	AVX (for XCR0 syncing during worlds switching). */
1076	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1077
1078	/* Intercept all DRx reads and writes by default. Changed later on. */
1079	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1080	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1081
1082	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1083	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1084
1085	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1086	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1087	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1088
1089	/* Set the IO permission bitmap physical addresses. */
1090	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1091
1092	/* LBR virtualization. */
1093	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1094
1095	/* The host ASID MBZ, for the guest start with 1. */
1096	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1097
1098	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1099	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1100
1101	/* Without Nested Paging, we need additionally intercepts. */
1102	if (!fNestedPaging)
1103	{
1104	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1105	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1106	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1107
1108	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1109	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1110	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1111
1112	/* Page faults must be intercepted to implement shadow paging. */
1113	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1114	}
1115
1116	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1117	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1118	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1119
1120	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1121	if (fUsePauseFilter)
1122	{
1123	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1124	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1125	if (fPauseFilterThreshold)
1126	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1127	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1128	}
1129
1130	/*
1131	* Setup the MSR permission bitmap.
1132	* The following MSRs are saved/restored automatically during the world-switch.
1133	* Don't intercept guest read/write accesses to these MSRs.
1134	*/
1135	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1136	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1137	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1140	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1141	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1142	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1143	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1144	{
1145	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1146	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1147	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1148	}
1149	else
1150	{
1151	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1152	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1153	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1154	}
1155	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1156
1157	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1158	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1159
1160	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1161	{
1162	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1163	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1164	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1165	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1166
1167	/* Copy the VMCB control area. */
1168	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1169
1170	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1171	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1172	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1173	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1174
1175	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1176	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1177
1178	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1179	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1180	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1181	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1182	}
1183
1184	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1185	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1186	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1187	#else
1188	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1189	#endif
1190	return VINF_SUCCESS;
1191	}
1192
1193
1194	/**
1195	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1196	*
1197	* @returns Pointer to the current context VMCB.
1198	* @param pVCpu The cross context virtual CPU structure.
1199	*/
1200	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1201	{
1202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1203	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1204	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1205	#endif
1206	return pVCpu->hmr0.s.svm.pVmcb;
1207	}
1208
1209
1210	/**
1211	* Gets a pointer to the nested-guest VMCB cache.
1212	*
1213	* @returns Pointer to the nested-guest VMCB cache.
1214	* @param pVCpu The cross context virtual CPU structure.
1215	*/
1216	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1217	{
1218	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1219	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1220	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1221	#else
1222	RT_NOREF(pVCpu);
1223	return NULL;
1224	#endif
1225	}
1226
1227
1228	/**
1229	* Invalidates a guest page by guest virtual address.
1230	*
1231	* @returns VBox status code.
1232	* @param pVCpu The cross context virtual CPU structure.
1233	* @param GCVirt Guest virtual address of the page to invalidate.
1234	*/
1235	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1236	{
1237	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1238
1239	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1240
1241	/* Skip it if a TLB flush is already pending. */
1242	if (!fFlushPending)
1243	{
1244	Log4Func(("%#RGv\n", GCVirt));
1245
1246	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1247	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1248
1249	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1251	}
1252	return VINF_SUCCESS;
1253	}
1254
1255
1256	/**
1257	* Flushes the appropriate tagged-TLB entries.
1258	*
1259	* @param pHostCpu The HM physical-CPU structure.
1260	* @param pVCpu The cross context virtual CPU structure.
1261	* @param pVmcb Pointer to the VM control block.
1262	*/
1263	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1264	{
1265	/*
1266	* Force a TLB flush for the first world switch if the current CPU differs from the one
1267	* we ran on last. This can happen both for start & resume due to long jumps back to
1268	* ring-3.
1269	*
1270	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1271	* correlation between it and the physical CPU.
1272	*
1273	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1274	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1275	*/
1276	bool fNewAsid = false;
1277	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1278	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1279	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1280	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1281	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1282	#endif
1283	)
1284	{
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1286	pVCpu->hmr0.s.fForceTLBFlush = true;
1287	fNewAsid = true;
1288	}
1289
1290	/* Set TLB flush state as checked until we return from the world switch. */
1291	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1292
1293	/* Check for explicit TLB flushes. */
1294	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1295	{
1296	pVCpu->hmr0.s.fForceTLBFlush = true;
1297	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1298	}
1299
1300	/*
1301	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1302	* This Host CPU requirement takes precedence.
1303	*/
1304	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1305	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1306	{
1307	pHostCpu->uCurrentAsid = 1;
1308	pVCpu->hmr0.s.uCurrentAsid = 1;
1309	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1310	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1311	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1312
1313	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1314	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1315	}
1316	else
1317	{
1318	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1319	if (pVCpu->hmr0.s.fForceTLBFlush)
1320	{
1321	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1322	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1323
1324	if (fNewAsid)
1325	{
1326	++pHostCpu->uCurrentAsid;
1327
1328	bool fHitASIDLimit = false;
1329	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1330	{
1331	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1332	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1333	fHitASIDLimit = true;
1334	}
1335
1336	if ( fHitASIDLimit
1337	\|\| pHostCpu->fFlushAsidBeforeUse)
1338	{
1339	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1340	pHostCpu->fFlushAsidBeforeUse = false;
1341	}
1342
1343	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1344	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1345	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1346	}
1347	else
1348	{
1349	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1350	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1351	else
1352	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1353	}
1354
1355	pVCpu->hmr0.s.fForceTLBFlush = false;
1356	}
1357	}
1358
1359	/* Update VMCB with the ASID. */
1360	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1361	{
1362	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1363	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1364	}
1365
1366	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1367	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1368	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1369	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1370	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1371	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1372	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1373	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1374
1375	#ifdef VBOX_WITH_STATISTICS
1376	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1377	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1378	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1379	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1380	{
1381	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1382	}
1383	else
1384	{
1385	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1386	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1387	}
1388	#endif
1389	}
1390
1391
1392	/**
1393	* Sets an exception intercept in the specified VMCB.
1394	*
1395	* @param pVmcb Pointer to the VM control block.
1396	* @param uXcpt The exception (X86_XCPT_*).
1397	*/
1398	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1399	{
1400	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1401	{
1402	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1403	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1404	}
1405	}
1406
1407
1408	/**
1409	* Clears an exception intercept in the specified VMCB.
1410	*
1411	* @param pVCpu The cross context virtual CPU structure.
1412	* @param pVmcb Pointer to the VM control block.
1413	* @param uXcpt The exception (X86_XCPT_*).
1414	*
1415	* @remarks This takes into account if we're executing a nested-guest and only
1416	* removes the exception intercept if both the guest -and- nested-guest
1417	* are not intercepting it.
1418	*/
1419	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1420	{
1421	Assert(uXcpt != X86_XCPT_DB);
1422	Assert(uXcpt != X86_XCPT_AC);
1423	Assert(uXcpt != X86_XCPT_GP);
1424	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1425	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1426	{
1427	bool fRemove = true;
1428	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1429	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1430	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1431	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1432	{
1433	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1434	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1435	}
1436	# else
1437	RT_NOREF(pVCpu);
1438	# endif
1439	if (fRemove)
1440	{
1441	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1442	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1443	}
1444	}
1445	#else
1446	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1447	#endif
1448	}
1449
1450
1451	/**
1452	* Sets a control intercept in the specified VMCB.
1453	*
1454	* @param pVmcb Pointer to the VM control block.
1455	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1456	*/
1457	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1458	{
1459	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1460	{
1461	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1462	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1463	}
1464	}
1465
1466
1467	/**
1468	* Clears a control intercept in the specified VMCB.
1469	*
1470	* @returns @c true if the intercept is still set, @c false otherwise.
1471	* @param pVCpu The cross context virtual CPU structure.
1472	* @param pVmcb Pointer to the VM control block.
1473	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1474	*
1475	* @remarks This takes into account if we're executing a nested-guest and only
1476	* removes the control intercept if both the guest -and- nested-guest
1477	* are not intercepting it.
1478	*/
1479	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1480	{
1481	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1482	{
1483	bool fRemove = true;
1484	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1485	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1486	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1487	{
1488	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1489	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1490	}
1491	#else
1492	RT_NOREF(pVCpu);
1493	#endif
1494	if (fRemove)
1495	{
1496	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1497	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1498	}
1499	}
1500
1501	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1502	}
1503
1504
1505	/**
1506	* Exports the guest (or nested-guest) CR0 into the VMCB.
1507	*
1508	* @param pVCpu The cross context virtual CPU structure.
1509	* @param pVmcb Pointer to the VM control block.
1510	*
1511	* @remarks This assumes we always pre-load the guest FPU.
1512	* @remarks No-long-jump zone!!!
1513	*/
1514	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1515	{
1516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1517
1518	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1519	uint64_t const uGuestCr0 = pCtx->cr0;
1520	uint64_t uShadowCr0 = uGuestCr0;
1521
1522	/* Always enable caching. */
1523	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1524
1525	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1526	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1527	{
1528	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1529	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1530	}
1531
1532	/*
1533	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1534	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1535	* see @bugref{7243#c103}.
1536	*/
1537	if (!(uGuestCr0 & X86_CR0_NE))
1538	{
1539	uShadowCr0 \|= X86_CR0_NE;
1540	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1541	}
1542	else
1543	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1544
1545	/*
1546	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1547	*
1548	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1549	* see @bugref{6944}.
1550	*
1551	* We also don't ever want to honor weird things like cache disable from the guest.
1552	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1553	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1554	*/
1555	if (uShadowCr0 == uGuestCr0)
1556	{
1557	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1558	{
1559	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1560	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1561	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1562	}
1563	else
1564	{
1565	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1566	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1567	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1568	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1569	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1570	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1571	}
1572	}
1573	else
1574	{
1575	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1576	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1577	}
1578	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1579
1580	Assert(!RT_HI_U32(uShadowCr0));
1581	if (pVmcb->guest.u64CR0 != uShadowCr0)
1582	{
1583	pVmcb->guest.u64CR0 = uShadowCr0;
1584	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1585	}
1586	}
1587
1588
1589	/**
1590	* Exports the guest (or nested-guest) CR3 into the VMCB.
1591	*
1592	* @param pVCpu The cross context virtual CPU structure.
1593	* @param pVmcb Pointer to the VM control block.
1594	*
1595	* @remarks No-long-jump zone!!!
1596	*/
1597	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1598	{
1599	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1600
1601	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1602	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1603	if (pVM->hmr0.s.fNestedPaging)
1604	{
1605	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1606	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1607	pVmcb->guest.u64CR3 = pCtx->cr3;
1608	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1609	}
1610	else
1611	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1612
1613	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1614	}
1615
1616
1617	/**
1618	* Exports the guest (or nested-guest) CR4 into the VMCB.
1619	*
1620	* @param pVCpu The cross context virtual CPU structure.
1621	* @param pVmcb Pointer to the VM control block.
1622	*
1623	* @remarks No-long-jump zone!!!
1624	*/
1625	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1626	{
1627	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1628
1629	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1630	uint64_t uShadowCr4 = pCtx->cr4;
1631	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1632	{
1633	switch (pVCpu->hm.s.enmShadowMode)
1634	{
1635	case PGMMODE_REAL:
1636	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1637	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1638
1639	case PGMMODE_32_BIT: /* 32-bit paging. */
1640	uShadowCr4 &= ~X86_CR4_PAE;
1641	break;
1642
1643	case PGMMODE_PAE: /* PAE paging. */
1644	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1645	/** Must use PAE paging as we could use physical memory > 4 GB */
1646	uShadowCr4 \|= X86_CR4_PAE;
1647	break;
1648
1649	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1650	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1651	#ifdef VBOX_WITH_64_BITS_GUESTS
1652	break;
1653	#else
1654	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1655	#endif
1656
1657	default: /* shut up gcc */
1658	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1659	}
1660	}
1661
1662	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1663	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1664	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1665	{
1666	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1667	hmR0SvmUpdateVmRunFunction(pVCpu);
1668	}
1669
1670	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1671	if (uShadowCr4 == pCtx->cr4)
1672	{
1673	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1674	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1675	else
1676	{
1677	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1678	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1679	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1680	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1681	}
1682	}
1683	else
1684	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1685
1686	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1687	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1688	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1689
1690	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1691	Assert(!RT_HI_U32(uShadowCr4));
1692	pVmcb->guest.u64CR4 = uShadowCr4;
1693	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1694
1695	return VINF_SUCCESS;
1696	}
1697
1698
1699	/**
1700	* Exports the guest (or nested-guest) control registers into the VMCB.
1701	*
1702	* @returns VBox status code.
1703	* @param pVCpu The cross context virtual CPU structure.
1704	* @param pVmcb Pointer to the VM control block.
1705	*
1706	* @remarks No-long-jump zone!!!
1707	*/
1708	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1709	{
1710	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1713	{
1714	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1715	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1716
1717	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1718	{
1719	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1720	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1721	}
1722
1723	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1724	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1725
1726	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1727	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1728	{
1729	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1730	if (RT_FAILURE(rc))
1731	return rc;
1732	}
1733
1734	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1735	}
1736	return VINF_SUCCESS;
1737	}
1738
1739
1740	/**
1741	* Exports the guest (or nested-guest) segment registers into the VMCB.
1742	*
1743	* @param pVCpu The cross context virtual CPU structure.
1744	* @param pVmcb Pointer to the VM control block.
1745	*
1746	* @remarks No-long-jump zone!!!
1747	*/
1748	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1749	{
1750	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1751	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1752
1753	/* Guest segment registers. */
1754	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1755	{
1756	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1757	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1760	{
1761	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1762	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1763	}
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1767
1768	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1769	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1770
1771	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1772	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1773
1774	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1775	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1776
1777	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1778	}
1779
1780	/* Guest TR. */
1781	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1782	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1783
1784	/* Guest LDTR. */
1785	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1786	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1787
1788	/* Guest GDTR. */
1789	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1790	{
1791	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1792	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1793	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1794	}
1795
1796	/* Guest IDTR. */
1797	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1798	{
1799	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1800	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1801	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1802	}
1803
1804	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1805	\| HM_CHANGED_GUEST_TABLE_MASK);
1806	}
1807
1808
1809	/**
1810	* Exports the guest (or nested-guest) MSRs into the VMCB.
1811	*
1812	* @param pVCpu The cross context virtual CPU structure.
1813	* @param pVmcb Pointer to the VM control block.
1814	*
1815	* @remarks No-long-jump zone!!!
1816	*/
1817	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1818	{
1819	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1820	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1821
1822	/* Guest Sysenter MSRs. */
1823	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1824	{
1825	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1826	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1827
1828	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1829	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1830
1831	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1832	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1833	}
1834
1835	/*
1836	* Guest EFER MSR.
1837	* AMD-V requires guest EFER.SVME to be set. Weird.
1838	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1839	*/
1840	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1841	{
1842	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1843	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1844	}
1845
1846	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1847	if ( !CPUMIsGuestInLongModeEx(pCtx)
1848	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1849	{
1850	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1851	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1852	}
1853
1854	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1855	{
1856	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1857	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1858	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1859	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1860	}
1861
1862	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1863	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1864
1865	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1866	\| HM_CHANGED_GUEST_EFER_MSR
1867	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1868	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1869
1870	/*
1871	* Setup the PAT MSR (applicable for Nested Paging only).
1872	*
1873	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1874	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1875	*
1876	* While guests can modify and see the modified values through the shadow values,
1877	* we shall not honor any guest modifications of this MSR to ensure caching is always
1878	* enabled similar to how we clear CR0.CD and NW bits.
1879	*
1880	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1881	*/
1882	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1883
1884	/* Enable the last branch record bit if LBR virtualization is enabled. */
1885	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1886	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1887	}
1888
1889
1890	/**
1891	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1892	* the necessary intercepts accordingly.
1893	*
1894	* @param pVCpu The cross context virtual CPU structure.
1895	* @param pVmcb Pointer to the VM control block.
1896	*
1897	* @remarks No-long-jump zone!!!
1898	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1899	*/
1900	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1901	{
1902	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1903
1904	/** @todo Figure out stepping with nested-guest. */
1905	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1906	{
1907	/*
1908	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1909	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1910	* Instead, they are strictly only requested when the nested hypervisor intercepts
1911	* them -- handled while merging VMCB controls.
1912	*
1913	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1914	* then the nested-guest debug state should be actively loaded on the host so that
1915	* nested-guest reads/writes its own debug registers without causing VM-exits.
1916	*/
1917	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1918	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1919	&& !CPUMIsGuestDebugStateActive(pVCpu))
1920	{
1921	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1922	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1923	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1924	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1925	}
1926
1927	pVmcb->guest.u64DR6 = pCtx->dr[6];
1928	pVmcb->guest.u64DR7 = pCtx->dr[7];
1929	return;
1930	}
1931
1932	/*
1933	* Anyone single stepping on the host side? If so, we'll have to use the
1934	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1935	* the VMM level like the VT-x implementations does.
1936	*/
1937	bool fInterceptMovDRx = false;
1938	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1939	if (fStepping)
1940	{
1941	pVCpu->hmr0.s.fClearTrapFlag = true;
1942	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1943	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1944	}
1945
1946	if ( fStepping
1947	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1948	{
1949	/*
1950	* Use the combined guest and host DRx values found in the hypervisor
1951	* register set because the debugger has breakpoints active or someone
1952	* is single stepping on the host side.
1953	*
1954	* Note! DBGF expects a clean DR6 state before executing guest code.
1955	*/
1956	if (!CPUMIsHyperDebugStateActive(pVCpu))
1957	{
1958	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1959	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1960	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1961	}
1962
1963	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1964	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1965	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1966	{
1967	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1968	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1969	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1970	}
1971
1972	/** @todo If we cared, we could optimize to allow the guest to read registers
1973	* with the same values. */
1974	fInterceptMovDRx = true;
1975	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1976	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1977	}
1978	else
1979	{
1980	/*
1981	* Update DR6, DR7 with the guest values if necessary.
1982	*/
1983	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1984	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1985	{
1986	pVmcb->guest.u64DR7 = pCtx->dr[7];
1987	pVmcb->guest.u64DR6 = pCtx->dr[6];
1988	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1989	}
1990	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1991
1992	/*
1993	* If the guest has enabled debug registers, we need to load them prior to
1994	* executing guest code so they'll trigger at the right time.
1995	*/
1996	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1997	{
1998	if (!CPUMIsGuestDebugStateActive(pVCpu))
1999	{
2000	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2001	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2002	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2003	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2004	}
2005	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2006	}
2007	/*
2008	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2009	* intercept #DB as DR6 is updated in the VMCB.
2010	*
2011	* Note! If we cared and dared, we could skip intercepting \#DB here.
2012	* However, \#DB shouldn't be performance critical, so we'll play safe
2013	* and keep the code similar to the VT-x code and always intercept it.
2014	*/
2015	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2016	fInterceptMovDRx = true;
2017	}
2018
2019	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2020	if (fInterceptMovDRx)
2021	{
2022	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2023	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2024	{
2025	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2026	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2027	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2028	}
2029	}
2030	else
2031	{
2032	if ( pVmcb->ctrl.u16InterceptRdDRx
2033	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2034	{
2035	pVmcb->ctrl.u16InterceptRdDRx = 0;
2036	pVmcb->ctrl.u16InterceptWrDRx = 0;
2037	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2038	}
2039	}
2040	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2041	}
2042
2043	/**
2044	* Exports the hardware virtualization state into the nested-guest
2045	* VMCB.
2046	*
2047	* @param pVCpu The cross context virtual CPU structure.
2048	* @param pVmcb Pointer to the VM control block.
2049	*
2050	* @remarks No-long-jump zone!!!
2051	*/
2052	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2053	{
2054	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2055
2056	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2057	{
2058	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2059	{
2060	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2061	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2062
2063	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2064	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2065	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2066	NOREF(pVM);
2067
2068	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2069	}
2070
2071	/*
2072	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2073	* since SVM doesn't have a preemption timer.
2074	*
2075	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2076	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2077	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2078	*/
2079	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2080	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2081	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2082	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2083	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2084	{
2085	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2086	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2087	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2088	}
2089	else
2090	{
2091	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2092	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2093	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2094	}
2095	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2096
2097	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2098	}
2099	}
2100
2101
2102	/**
2103	* Exports the guest APIC TPR state into the VMCB.
2104	*
2105	* @returns VBox status code.
2106	* @param pVCpu The cross context virtual CPU structure.
2107	* @param pVmcb Pointer to the VM control block.
2108	*/
2109	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2110	{
2111	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2112
2113	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2114	{
2115	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2116	if ( PDMHasApic(pVM)
2117	&& PDMApicIsEnabled(pVCpu))
2118	{
2119	bool fPendingIntr;
2120	uint8_t u8Tpr;
2121	int rc = PDMApicGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2122	AssertRCReturn(rc, rc);
2123
2124	/* Assume that we need to trap all TPR accesses and thus need not check on
2125	every #VMEXIT if we should update the TPR. */
2126	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2127	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2128
2129	if (!pVM->hm.s.fTprPatchingActive)
2130	{
2131	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2132	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2133
2134	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2135	can deliver the interrupt to the guest. */
2136	if (fPendingIntr)
2137	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2138	else
2139	{
2140	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2141	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2142	}
2143
2144	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2145	}
2146	else
2147	{
2148	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2149	pVmcb->guest.u64LSTAR = u8Tpr;
2150	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2151
2152	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2153	if (fPendingIntr)
2154	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2155	else
2156	{
2157	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2158	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2159	}
2160	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2161	}
2162	}
2163	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2164	}
2165	return VINF_SUCCESS;
2166	}
2167
2168
2169	/**
2170	* Sets up the exception interrupts required for guest execution in the VMCB.
2171	*
2172	* @param pVCpu The cross context virtual CPU structure.
2173	* @param pVmcb Pointer to the VM control block.
2174	*
2175	* @remarks No-long-jump zone!!!
2176	*/
2177	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2178	{
2179	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2180
2181	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2182	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2183	{
2184	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2185	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2186	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2187	else
2188	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2189
2190	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2191	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
2192	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2193	else
2194	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2195
2196	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2197	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2198	}
2199	}
2200
2201
2202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2203	/**
2204	* Merges guest and nested-guest intercepts for executing the nested-guest using
2205	* hardware-assisted SVM.
2206	*
2207	* This merges the guest and nested-guest intercepts in a way that if the outer
2208	* guest intercept is set we need to intercept it in the nested-guest as
2209	* well.
2210	*
2211	* @param pVCpu The cross context virtual CPU structure.
2212	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2213	*/
2214	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2215	{
2216	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2217	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2218	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2219	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2220
2221	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2222	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2223	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2224
2225	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2226	XCR0 syncing during worlds switching). */
2227	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2228
2229	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2230	if (!pVM->hmr0.s.fNestedPaging)
2231	{
2232	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2233	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2234	}
2235
2236	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2237	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2238	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2239
2240	/*
2241	* Merge the guest's exception intercepts into the nested-guest VMCB.
2242	*
2243	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2244	* while executing the nested-guest.
2245	*
2246	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2247	* be tweaked later depending on how we wish to implement breakpoints.
2248	*
2249	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2250	* loaded into their guests, not ours.
2251	*
2252	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2253	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2254	*/
2255	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2256	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2257	& ~( RT_BIT(X86_XCPT_UD)
2258	\| RT_BIT(X86_XCPT_BP)
2259	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2260	#else
2261	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2262	#endif
2263
2264	/*
2265	* Adjust intercepts while executing the nested-guest that differ from the
2266	* outer guest intercepts.
2267	*
2268	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2269	* that belong to the nested-guest to the outer guest.
2270	*
2271	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2272	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2273	*/
2274	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2275	\| SVM_CTRL_INTERCEPT_VMMCALL))
2276	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2277
2278	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2279	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2280
2281	/* Finally, update the VMCB clean bits. */
2282	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2283	}
2284	#endif
2285
2286
2287	/**
2288	* Enters the AMD-V session.
2289	*
2290	* @returns VBox status code.
2291	* @param pVCpu The cross context virtual CPU structure.
2292	*/
2293	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2294	{
2295	AssertPtr(pVCpu);
2296	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2297	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2298
2299	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2300	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2301	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2302
2303	pVCpu->hmr0.s.fLeaveDone = false;
2304	return VINF_SUCCESS;
2305	}
2306
2307
2308	/**
2309	* Thread-context callback for AMD-V.
2310	*
2311	* This is used together with RTThreadCtxHookCreate() on platforms which
2312	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2313	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2314	*
2315	* @param enmEvent The thread-context event.
2316	* @param pVCpu The cross context virtual CPU structure.
2317	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2318	* @thread EMT(pVCpu)
2319	*/
2320	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2321	{
2322	NOREF(fGlobalInit);
2323
2324	switch (enmEvent)
2325	{
2326	case RTTHREADCTXEVENT_OUT:
2327	{
2328	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2329	VMCPU_ASSERT_EMT(pVCpu);
2330
2331	/* No longjmps (log-flush, locks) in this fragile context. */
2332	VMMRZCallRing3Disable(pVCpu);
2333
2334	if (!pVCpu->hmr0.s.fLeaveDone)
2335	{
2336	hmR0SvmLeave(pVCpu, false /* fImportState */);
2337	pVCpu->hmr0.s.fLeaveDone = true;
2338	}
2339
2340	/* Leave HM context, takes care of local init (term). */
2341	int rc = HMR0LeaveCpu(pVCpu);
2342	AssertRC(rc); NOREF(rc);
2343
2344	/* Restore longjmp state. */
2345	VMMRZCallRing3Enable(pVCpu);
2346	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2347	break;
2348	}
2349
2350	case RTTHREADCTXEVENT_IN:
2351	{
2352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2353	VMCPU_ASSERT_EMT(pVCpu);
2354
2355	/* No longjmps (log-flush, locks) in this fragile context. */
2356	VMMRZCallRing3Disable(pVCpu);
2357
2358	/*
2359	* Initialize the bare minimum state required for HM. This takes care of
2360	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2361	*/
2362	int rc = hmR0EnterCpu(pVCpu);
2363	AssertRC(rc); NOREF(rc);
2364	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2365	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2366
2367	pVCpu->hmr0.s.fLeaveDone = false;
2368
2369	/* Restore longjmp state. */
2370	VMMRZCallRing3Enable(pVCpu);
2371	break;
2372	}
2373
2374	default:
2375	break;
2376	}
2377	}
2378
2379
2380	/**
2381	* Saves the host state.
2382	*
2383	* @returns VBox status code.
2384	* @param pVCpu The cross context virtual CPU structure.
2385	*
2386	* @remarks No-long-jump zone!!!
2387	*/
2388	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2389	{
2390	NOREF(pVCpu);
2391
2392	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2393	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2394	return VINF_SUCCESS;
2395	}
2396
2397
2398	/**
2399	* Exports the guest or nested-guest state from the virtual-CPU context into the
2400	* VMCB.
2401	*
2402	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2403	* code based on the virtual-CPU mode.
2404	*
2405	* @returns VBox status code.
2406	* @param pVCpu The cross context virtual CPU structure.
2407	* @param pSvmTransient Pointer to the SVM-transient structure.
2408	*
2409	* @remarks No-long-jump zone!!!
2410	*/
2411	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2412	{
2413	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2414
2415	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2416	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2417	Assert(pVmcb);
2418
2419	pVmcb->guest.u64RIP = pCtx->rip;
2420	pVmcb->guest.u64RSP = pCtx->rsp;
2421	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2422	pVmcb->guest.u64RAX = pCtx->rax;
2423
2424	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2425	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2426
2427	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2428	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2429	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2430	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2431	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2432
2433	ASMSetFlags(fEFlags);
2434
2435	if (!fIsNestedGuest)
2436	{
2437	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2438	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2439	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2440	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2441	}
2442
2443	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2444	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2445	\| HM_CHANGED_GUEST_RFLAGS
2446	\| HM_CHANGED_GUEST_GPRS_MASK
2447	\| HM_CHANGED_GUEST_X87
2448	\| HM_CHANGED_GUEST_SSE_AVX
2449	\| HM_CHANGED_GUEST_OTHER_XSAVE
2450	\| HM_CHANGED_GUEST_XCRx
2451	\| HM_CHANGED_GUEST_TSC_AUX
2452	\| HM_CHANGED_GUEST_OTHER_MSRS;
2453	if (fIsNestedGuest)
2454	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2455	\| HM_CHANGED_GUEST_APIC_TPR;
2456
2457	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2458	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2459
2460	#ifdef VBOX_STRICT
2461	/*
2462	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2463	* except for the host-context and/or shared host-guest context bits.
2464	*/
2465	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2466	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2467	("fCtxChanged=%#RX64\n", fCtxChanged));
2468
2469	/*
2470	* If we need to log state that isn't always imported, we'll need to import them here.
2471	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2472	*/
2473	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2474	#endif
2475
2476	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2477	return VINF_SUCCESS;
2478	}
2479
2480	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2481
2482	/**
2483	* Merges the guest and nested-guest MSR permission bitmap.
2484	*
2485	* If the guest is intercepting an MSR we need to intercept it regardless of
2486	* whether the nested-guest is intercepting it or not.
2487	*
2488	* @param pHostCpu The HM physical-CPU structure.
2489	* @param pVCpu The cross context virtual CPU structure.
2490	*
2491	* @remarks No-long-jmp zone!!!
2492	*/
2493	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2494	{
2495	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2496	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2497	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2498
2499	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2500	uint32_t const offRsvdQwords = 0x1800 >> 3;
2501	for (uint32_t i = 0; i < offRsvdQwords; i++)
2502	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2503	}
2504
2505
2506	/**
2507	* Caches the nested-guest VMCB fields before we modify them for execution using
2508	* hardware-assisted SVM.
2509	*
2510	* @returns true if the VMCB was previously already cached, false otherwise.
2511	* @param pVCpu The cross context virtual CPU structure.
2512	*
2513	* @sa HMNotifySvmNstGstVmexit.
2514	*/
2515	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2516	{
2517	/*
2518	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2519	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2520	*
2521	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2522	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2523	*/
2524	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2525	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2526	if (!fWasCached)
2527	{
2528	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2529	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2530	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2531	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2532	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2533	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2534	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2535	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2536	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2537	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2538	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2539	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2540	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2541	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2542	pVmcbNstGstCache->fCacheValid = true;
2543	Log4Func(("Cached VMCB fields\n"));
2544	}
2545
2546	return fWasCached;
2547	}
2548
2549
2550	/**
2551	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2552	*
2553	* This is done the first time we enter nested-guest execution using SVM R0
2554	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2555	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2556	*
2557	* @param pVCpu The cross context virtual CPU structure.
2558	*/
2559	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2560	{
2561	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2562	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2563
2564	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2565
2566	/*
2567	* First cache the nested-guest VMCB fields we may potentially modify.
2568	*/
2569	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2570	if (!fVmcbCached)
2571	{
2572	/*
2573	* The IOPM of the nested-guest can be ignored because the the guest always
2574	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2575	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2576	*/
2577	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2578
2579	/*
2580	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2581	* nested-paging suddenly while executing a VM (see assertion at the end of
2582	* Trap0eHandler() in PGMAllBth.h).
2583	*/
2584	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2585
2586	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2587	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2588
2589	/*
2590	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2591	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2592	*/
2593	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2594
2595	# ifdef DEBUG_ramshankar
2596	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2597	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2598	# endif
2599
2600	/*
2601	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2602	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2603	*/
2604	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2605	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2606	\| SVM_CTRL_INTERCEPT_VMLOAD;
2607
2608	/*
2609	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2610	* CLGI/STGI instructions executed by the nested-guest.
2611	*/
2612	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2613	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2614	\| SVM_CTRL_INTERCEPT_STGI;
2615
2616	/* Merge the guest and nested-guest intercepts. */
2617	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2618
2619	/* Update the VMCB clean bits. */
2620	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2621	}
2622	else
2623	{
2624	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2625	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2626	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2627	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2628	}
2629	}
2630
2631	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2632
2633	/**
2634	* Exports the state shared between the host and guest (or nested-guest) into
2635	* the VMCB.
2636	*
2637	* @param pVCpu The cross context virtual CPU structure.
2638	* @param pVmcb Pointer to the VM control block.
2639	*
2640	* @remarks No-long-jump zone!!!
2641	*/
2642	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2643	{
2644	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2645	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2646
2647	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2648	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2649
2650	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2651	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2652	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2653	}
2654
2655
2656	/**
2657	* Worker for SVMR0ImportStateOnDemand.
2658	*
2659	* @param pVCpu The cross context virtual CPU structure.
2660	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2661	*/
2662	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2663	{
2664	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2665
2666	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2667	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2668	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2669	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2670
2671	/*
2672	* We disable interrupts to make the updating of the state and in particular
2673	* the fExtrn modification atomic wrt to preemption hooks.
2674	*/
2675	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2676
2677	fWhat &= pCtx->fExtrn;
2678	if (fWhat)
2679	{
2680	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2681	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2682	{
2683	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2684	{
2685	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2686	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2687	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2688	}
2689	}
2690
2691	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2692	{
2693	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2694	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2695	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2696	}
2697	#endif
2698
2699	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2700	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2701
2702	if (fWhat & CPUMCTX_EXTRN_RIP)
2703	pCtx->rip = pVmcbGuest->u64RIP;
2704
2705	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2706	{
2707	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2708	if (pVCpu->hmr0.s.fClearTrapFlag)
2709	{
2710	pVCpu->hmr0.s.fClearTrapFlag = false;
2711	pCtx->eflags.Bits.u1TF = 0;
2712	}
2713	}
2714
2715	if (fWhat & CPUMCTX_EXTRN_RSP)
2716	pCtx->rsp = pVmcbGuest->u64RSP;
2717
2718	if (fWhat & CPUMCTX_EXTRN_RAX)
2719	pCtx->rax = pVmcbGuest->u64RAX;
2720
2721	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2722	{
2723	if (fWhat & CPUMCTX_EXTRN_CS)
2724	{
2725	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2726	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2727	/** @todo SELM might need to be fixed as it too should not care about the
2728	* granularity bit. See @bugref{6785}. */
2729	if ( !pCtx->cs.Attr.n.u1Granularity
2730	&& pCtx->cs.Attr.n.u1Present
2731	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2732	{
2733	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2734	pCtx->cs.Attr.n.u1Granularity = 1;
2735	}
2736	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2737	}
2738	if (fWhat & CPUMCTX_EXTRN_SS)
2739	{
2740	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2741	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2742	/*
2743	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2744	* VMCB and uses that and thus it's possible that when the CPL changes during
2745	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2746	* AMD Fusion CPUs with 64-bit guests.
2747	*
2748	* See AMD spec. 15.5.1 "Basic operation".
2749	*/
2750	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2751	uint8_t const uCpl = pVmcbGuest->u8CPL;
2752	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2753	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2754	}
2755	if (fWhat & CPUMCTX_EXTRN_DS)
2756	{
2757	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2758	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2759	}
2760	if (fWhat & CPUMCTX_EXTRN_ES)
2761	{
2762	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2763	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2764	}
2765	if (fWhat & CPUMCTX_EXTRN_FS)
2766	{
2767	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2768	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2769	}
2770	if (fWhat & CPUMCTX_EXTRN_GS)
2771	{
2772	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2773	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2774	}
2775	}
2776
2777	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2778	{
2779	if (fWhat & CPUMCTX_EXTRN_TR)
2780	{
2781	/*
2782	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2783	* are used between Intel and AMD, see @bugref{6208#c39}.
2784	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2785	*/
2786	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2787	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2788	{
2789	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2790	\|\| CPUMIsGuestInLongModeEx(pCtx))
2791	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2792	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2793	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2794	}
2795	}
2796
2797	if (fWhat & CPUMCTX_EXTRN_LDTR)
2798	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2799
2800	if (fWhat & CPUMCTX_EXTRN_GDTR)
2801	{
2802	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2803	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2804	}
2805
2806	if (fWhat & CPUMCTX_EXTRN_IDTR)
2807	{
2808	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2809	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2810	}
2811	}
2812
2813	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2814	{
2815	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2816	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2817	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2818	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2819	}
2820
2821	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2822	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2823	{
2824	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2825	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2826	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2827	}
2828
2829	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2830	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2831
2832	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2833	{
2834	if (fWhat & CPUMCTX_EXTRN_DR6)
2835	{
2836	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2837	pCtx->dr[6] = pVmcbGuest->u64DR6;
2838	else
2839	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2840	}
2841
2842	if (fWhat & CPUMCTX_EXTRN_DR7)
2843	{
2844	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2845	pCtx->dr[7] = pVmcbGuest->u64DR7;
2846	else
2847	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2848	}
2849	}
2850
2851	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2852	{
2853	if (fWhat & CPUMCTX_EXTRN_CR0)
2854	{
2855	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2856	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2857	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2858	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2859	CPUMSetGuestCR0(pVCpu, uCr0);
2860	VMMRZCallRing3Enable(pVCpu);
2861	}
2862
2863	if (fWhat & CPUMCTX_EXTRN_CR2)
2864	pCtx->cr2 = pVmcbGuest->u64CR2;
2865
2866	if (fWhat & CPUMCTX_EXTRN_CR3)
2867	{
2868	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2869	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2870	{
2871	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2872	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2873	}
2874	}
2875
2876	/* Changes to CR4 are always intercepted. */
2877	}
2878
2879	/* Update fExtrn. */
2880	pCtx->fExtrn &= ~fWhat;
2881
2882	/* If everything has been imported, clear the HM keeper bit. */
2883	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2884	{
2885	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2886	Assert(!pCtx->fExtrn);
2887	}
2888	}
2889	else
2890	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2891
2892	ASMSetFlags(fEFlags);
2893
2894	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2895
2896	/*
2897	* Honor any pending CR3 updates.
2898	*
2899	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2900	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2901	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2902	* handling -> hmR0SvmImportGuestState() and here we are.
2903	*
2904	* The reason for such complicated handling is because VM-exits that call into PGM expect
2905	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2906	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2907	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2908	*
2909	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2910	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2911	*/
2912	if ( VMMRZCallRing3IsEnabled(pVCpu)
2913	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2914	{
2915	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2916	PGMUpdateCR3(pVCpu, pCtx->cr3);
2917	}
2918	}
2919
2920
2921	/**
2922	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2923	* context.
2924	*
2925	* Currently there is no residual state left in the CPU that is not updated in the
2926	* VMCB.
2927	*
2928	* @returns VBox status code.
2929	* @param pVCpu The cross context virtual CPU structure.
2930	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2931	*/
2932	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2933	{
2934	hmR0SvmImportGuestState(pVCpu, fWhat);
2935	return VINF_SUCCESS;
2936	}
2937
2938
2939	/**
2940	* Gets SVM \#VMEXIT auxiliary information.
2941	*
2942	* @returns VBox status code.
2943	* @param pVCpu The cross context virtual CPU structure.
2944	* @param pSvmExitAux Where to store the auxiliary info.
2945	*/
2946	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2947	{
2948	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2949	if (RT_LIKELY(pSvmTransient))
2950	{
2951	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2952	if (RT_LIKELY(pVmcb))
2953	{
2954	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2955	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2956	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2957	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2958	return VINF_SUCCESS;
2959	}
2960	return VERR_SVM_IPE_5;
2961	}
2962	return VERR_NOT_AVAILABLE;
2963	}
2964
2965
2966	/**
2967	* Does the necessary state syncing before returning to ring-3 for any reason
2968	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2969	*
2970	* @param pVCpu The cross context virtual CPU structure.
2971	* @param fImportState Whether to import the guest state from the VMCB back
2972	* to the guest-CPU context.
2973	*
2974	* @remarks No-long-jmp zone!!!
2975	*/
2976	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2977	{
2978	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2979	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2980
2981	/*
2982	* !!! IMPORTANT !!!
2983	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2984	*/
2985
2986	/* Save the guest state if necessary. */
2987	if (fImportState)
2988	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2989
2990	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2991	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2992	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2993
2994	/*
2995	* Restore host debug registers if necessary and resync on next R0 reentry.
2996	*/
2997	#ifdef VBOX_STRICT
2998	if (CPUMIsHyperDebugStateActive(pVCpu))
2999	{
3000	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
3001	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
3002	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
3003	}
3004	#endif
3005	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3006	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3007	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3008
3009	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3010	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3011	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3012	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3013	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3014	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3015	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3016
3017	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3018	}
3019
3020
3021	/**
3022	* Leaves the AMD-V session.
3023	*
3024	* Only used while returning to ring-3 either due to longjump or exits to
3025	* ring-3.
3026	*
3027	* @returns VBox status code.
3028	* @param pVCpu The cross context virtual CPU structure.
3029	*/
3030	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3031	{
3032	HM_DISABLE_PREEMPT(pVCpu);
3033	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3034	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3035
3036	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3037	and done this from the SVMR0ThreadCtxCallback(). */
3038	if (!pVCpu->hmr0.s.fLeaveDone)
3039	{
3040	hmR0SvmLeave(pVCpu, true /* fImportState */);
3041	pVCpu->hmr0.s.fLeaveDone = true;
3042	}
3043
3044	/*
3045	* !!! IMPORTANT !!!
3046	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3047	*/
3048
3049	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3050	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3051	VMMR0ThreadCtxHookDisable(pVCpu);
3052
3053	/* Leave HM context. This takes care of local init (term). */
3054	int rc = HMR0LeaveCpu(pVCpu);
3055
3056	HM_RESTORE_PREEMPT();
3057	return rc;
3058	}
3059
3060
3061	/**
3062	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3063	* any remaining host state) before we go back to ring-3 due to an assertion.
3064	*
3065	* @param pVCpu The cross context virtual CPU structure.
3066	*/
3067	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3068	{
3069	/*
3070	* !!! IMPORTANT !!!
3071	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3072	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3073	*/
3074	VMMR0AssertionRemoveNotification(pVCpu);
3075	VMMRZCallRing3Disable(pVCpu);
3076	HM_DISABLE_PREEMPT(pVCpu);
3077
3078	/* Import the entire guest state. */
3079	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3080
3081	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3082	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3083
3084	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3085	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3086
3087	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3088	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3089	VMMR0ThreadCtxHookDisable(pVCpu);
3090
3091	/* Leave HM context. This takes care of local init (term). */
3092	HMR0LeaveCpu(pVCpu);
3093
3094	HM_RESTORE_PREEMPT();
3095	return VINF_SUCCESS;
3096	}
3097
3098
3099	/**
3100	* Take necessary actions before going back to ring-3.
3101	*
3102	* An action requires us to go back to ring-3. This function does the necessary
3103	* steps before we can safely return to ring-3. This is not the same as longjmps
3104	* to ring-3, this is voluntary.
3105	*
3106	* @returns Strict VBox status code.
3107	* @param pVCpu The cross context virtual CPU structure.
3108	* @param rcExit The reason for exiting to ring-3. Can be
3109	* VINF_VMM_UNKNOWN_RING3_CALL.
3110	*/
3111	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3112	{
3113	Assert(pVCpu);
3114	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3115
3116	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3117	VMMRZCallRing3Disable(pVCpu);
3118	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3119	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3120
3121	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3122	if (pVCpu->hm.s.Event.fPending)
3123	{
3124	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3125	Assert(!pVCpu->hm.s.Event.fPending);
3126	}
3127
3128	/* Sync. the necessary state for going back to ring-3. */
3129	hmR0SvmLeaveSession(pVCpu);
3130	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3131
3132	/* Thread-context hooks are unregistered at this point!!! */
3133	/* Ring-3 callback notifications are unregistered at this point!!! */
3134
3135	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3136	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3137	\| CPUM_CHANGED_LDTR
3138	\| CPUM_CHANGED_GDTR
3139	\| CPUM_CHANGED_IDTR
3140	\| CPUM_CHANGED_TR
3141	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3142	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3143	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3144	{
3145	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3146	}
3147
3148	/* Update the exit-to-ring 3 reason. */
3149	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3150
3151	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3152	if ( rcExit != VINF_EM_RAW_INTERRUPT
3153	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3154	{
3155	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3156	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3157	}
3158
3159	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3160	VMMRZCallRing3Enable(pVCpu);
3161
3162	/*
3163	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3164	* and if we're injecting an event we should have a TRPM trap pending.
3165	*/
3166	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3167	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3168	VERR_SVM_IPE_5);
3169	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3170	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3171	VERR_SVM_IPE_4);
3172
3173	return rcExit;
3174	}
3175
3176
3177	/**
3178	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3179	* intercepts.
3180	*
3181	* @param pVCpu The cross context virtual CPU structure.
3182	* @param pVmcb Pointer to the VM control block.
3183	*
3184	* @remarks No-long-jump zone!!!
3185	*/
3186	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3187	{
3188	/*
3189	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3190	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3191	* RDTSC/RDTSCP as well.
3192	*/
3193	bool fParavirtTsc;
3194	uint64_t uTscOffset;
3195	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3196
3197	bool fIntercept;
3198	if (fCanUseRealTsc)
3199	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3200	else
3201	{
3202	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3203	fIntercept = true;
3204	}
3205
3206	if (!fIntercept)
3207	{
3208	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3209	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3210	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3211	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3212	#endif
3213
3214	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3215	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3216	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3217	}
3218
3219	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3220	information before every VM-entry, hence we have nothing to do here at the moment. */
3221	if (fParavirtTsc)
3222	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3223	}
3224
3225
3226	/**
3227	* Sets an event as a pending event to be injected into the guest.
3228	*
3229	* @param pVCpu The cross context virtual CPU structure.
3230	* @param pEvent Pointer to the SVM event.
3231	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3232	* page-fault.
3233	*
3234	* @remarks Statistics counter assumes this is a guest event being reflected to
3235	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3236	*/
3237	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3238	{
3239	Assert(!pVCpu->hm.s.Event.fPending);
3240	Assert(pEvent->n.u1Valid);
3241
3242	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3243	pVCpu->hm.s.Event.fPending = true;
3244	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3245
3246	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3247	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3248	}
3249
3250
3251	/**
3252	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3253	*
3254	* @param pVCpu The cross context virtual CPU structure.
3255	*/
3256	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3257	{
3258	SVMEVENT Event;
3259	Event.u = 0;
3260	Event.n.u1Valid = 1;
3261	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3262	Event.n.u8Vector = X86_XCPT_DE;
3263	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3264	}
3265
3266
3267	/**
3268	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3269	*
3270	* @param pVCpu The cross context virtual CPU structure.
3271	*/
3272	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3273	{
3274	SVMEVENT Event;
3275	Event.u = 0;
3276	Event.n.u1Valid = 1;
3277	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3278	Event.n.u8Vector = X86_XCPT_UD;
3279	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3280	}
3281
3282
3283	/**
3284	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3285	*
3286	* @param pVCpu The cross context virtual CPU structure.
3287	*/
3288	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3289	{
3290	SVMEVENT Event;
3291	Event.u = 0;
3292	Event.n.u1Valid = 1;
3293	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3294	Event.n.u8Vector = X86_XCPT_DB;
3295	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3296	}
3297
3298
3299	/**
3300	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3301	*
3302	* @param pVCpu The cross context virtual CPU structure.
3303	* @param u32ErrCode The error-code for the page-fault.
3304	* @param uFaultAddress The page fault address (CR2).
3305	*
3306	* @remarks This updates the guest CR2 with @a uFaultAddress!
3307	*/
3308	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3309	{
3310	SVMEVENT Event;
3311	Event.u = 0;
3312	Event.n.u1Valid = 1;
3313	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3314	Event.n.u8Vector = X86_XCPT_PF;
3315	Event.n.u1ErrorCodeValid = 1;
3316	Event.n.u32ErrorCode = u32ErrCode;
3317
3318	/* Update CR2 of the guest. */
3319	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3320	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3321	{
3322	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3323	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3324	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3325	}
3326
3327	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3328	}
3329
3330
3331	/**
3332	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3333	*
3334	* @param pVCpu The cross context virtual CPU structure.
3335	*/
3336	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3337	{
3338	SVMEVENT Event;
3339	Event.u = 0;
3340	Event.n.u1Valid = 1;
3341	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3342	Event.n.u8Vector = X86_XCPT_MF;
3343	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3344	}
3345
3346
3347	/**
3348	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3349	*
3350	* @param pVCpu The cross context virtual CPU structure.
3351	*/
3352	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3353	{
3354	SVMEVENT Event;
3355	Event.u = 0;
3356	Event.n.u1Valid = 1;
3357	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3358	Event.n.u8Vector = X86_XCPT_DF;
3359	Event.n.u1ErrorCodeValid = 1;
3360	Event.n.u32ErrorCode = 0;
3361	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3362	}
3363
3364
3365	/**
3366	* Injects an event into the guest upon VMRUN by updating the relevant field
3367	* in the VMCB.
3368	*
3369	* @param pVCpu The cross context virtual CPU structure.
3370	* @param pVmcb Pointer to the guest VM control block.
3371	* @param pEvent Pointer to the event.
3372	*
3373	* @remarks No-long-jump zone!!!
3374	* @remarks Requires CR0!
3375	*/
3376	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3377	{
3378	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3379	pVmcb->ctrl.EventInject.u = pEvent->u;
3380	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3381	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3382	{
3383	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3384	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3385	}
3386	else
3387	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3388	RT_NOREF(pVCpu);
3389
3390	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3391	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3392	}
3393
3394
3395
3396	/**
3397	* Converts any TRPM trap into a pending HM event. This is typically used when
3398	* entering from ring-3 (not longjmp returns).
3399	*
3400	* @param pVCpu The cross context virtual CPU structure.
3401	*/
3402	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3403	{
3404	Assert(TRPMHasTrap(pVCpu));
3405	Assert(!pVCpu->hm.s.Event.fPending);
3406
3407	uint8_t uVector;
3408	TRPMEVENT enmTrpmEvent;
3409	uint32_t uErrCode;
3410	RTGCUINTPTR GCPtrFaultAddress;
3411	uint8_t cbInstr;
3412
3413	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3414	AssertRC(rc);
3415
3416	SVMEVENT Event;
3417	Event.u = 0;
3418	Event.n.u1Valid = 1;
3419	Event.n.u8Vector = uVector;
3420
3421	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3422	if (enmTrpmEvent == TRPM_TRAP)
3423	{
3424	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3425	switch (uVector)
3426	{
3427	case X86_XCPT_BP:
3428	case X86_XCPT_OF:
3429	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3430	RT_FALL_THRU();
3431
3432	case X86_XCPT_PF:
3433	case X86_XCPT_DF:
3434	case X86_XCPT_TS:
3435	case X86_XCPT_NP:
3436	case X86_XCPT_SS:
3437	case X86_XCPT_GP:
3438	case X86_XCPT_AC:
3439	{
3440	Event.n.u1ErrorCodeValid = 1;
3441	Event.n.u32ErrorCode = uErrCode;
3442	break;
3443	}
3444	}
3445	}
3446	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3447	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3448	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3449	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3450	else if (enmTrpmEvent == TRPM_NMI)
3451	Event.n.u3Type = SVM_EVENT_NMI;
3452	else
3453	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3454
3455	rc = TRPMResetTrap(pVCpu);
3456	AssertRC(rc);
3457
3458	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3459	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3460
3461	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3462	}
3463
3464
3465	/**
3466	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3467	* AMD-V to execute any instruction.
3468	*
3469	* @param pVCpu The cross context virtual CPU structure.
3470	*/
3471	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3472	{
3473	Assert(pVCpu->hm.s.Event.fPending);
3474	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3475
3476	SVMEVENT Event;
3477	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3478
3479	uint8_t uVector = Event.n.u8Vector;
3480	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3481
3482	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3483
3484	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3485	AssertRC(rc);
3486
3487	if (Event.n.u1ErrorCodeValid)
3488	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3489
3490	if ( enmTrapType == TRPM_TRAP
3491	&& uVector == X86_XCPT_PF)
3492	{
3493	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3494	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3495	}
3496	else if (enmTrapType == TRPM_SOFTWARE_INT)
3497	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3498	pVCpu->hm.s.Event.fPending = false;
3499	}
3500
3501
3502	/**
3503	* Sets the virtual interrupt intercept control in the VMCB.
3504	*
3505	* @param pVCpu The cross context virtual CPU structure.
3506	* @param pVmcb Pointer to the VM control block.
3507	*/
3508	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3509	{
3510	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3511
3512	/*
3513	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3514	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3515	* (updating ISR at the right time) and inject the interrupt.
3516	*
3517	* With AVIC is supported, we could make use of the asynchronously delivery without
3518	* #VMEXIT and we would be passing the AVIC page to SVM.
3519	*
3520	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3521	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3522	*/
3523	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3524	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3525	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3526	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3527	Log4(("Set VINTR intercept\n"));
3528	}
3529
3530
3531	/**
3532	* Clears the virtual interrupt intercept control in the VMCB as
3533	* we are figured the guest is unable process any interrupts
3534	* at this point of time.
3535	*
3536	* @param pVCpu The cross context virtual CPU structure.
3537	* @param pVmcb Pointer to the VM control block.
3538	*/
3539	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3540	{
3541	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3542
3543	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3544	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3545	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3546	{
3547	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3548	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3549	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3550	Log4(("Cleared VINTR intercept\n"));
3551	}
3552	}
3553
3554
3555	/**
3556	* Evaluates the event to be delivered to the guest and sets it as the pending
3557	* event.
3558	*
3559	* @returns Strict VBox status code.
3560	* @param pVCpu The cross context virtual CPU structure.
3561	* @param pSvmTransient Pointer to the SVM transient structure.
3562	*/
3563	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3564	{
3565	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3566	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3567	\| CPUMCTX_EXTRN_RFLAGS
3568	\| CPUMCTX_EXTRN_INHIBIT_INT
3569	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3570
3571	Assert(!pVCpu->hm.s.Event.fPending);
3572	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3573	Assert(pVmcb);
3574
3575	bool const fGif = CPUMGetGuestGif(pCtx);
3576	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3577	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3578
3579	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3580	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3581	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3582
3583	/** @todo SMI. SMIs take priority over NMIs. */
3584
3585	/*
3586	* Check if the guest or nested-guest can receive NMIs.
3587	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3588	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3589	*/
3590	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3591	&& !fBlockNmi)
3592	{
3593	if ( fGif
3594	&& !fIntShadow)
3595	{
3596	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3597	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3598	{
3599	Log4(("Intercepting NMI -> #VMEXIT\n"));
3600	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3601	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3602	}
3603	#endif
3604	Log4(("Setting NMI pending for injection\n"));
3605	SVMEVENT Event;
3606	Event.u = 0;
3607	Event.n.u1Valid = 1;
3608	Event.n.u8Vector = X86_XCPT_NMI;
3609	Event.n.u3Type = SVM_EVENT_NMI;
3610	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3611	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3612	}
3613	else if (!fGif)
3614	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3615	else if (!pSvmTransient->fIsNestedGuest)
3616	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3617	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3618	}
3619	/*
3620	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3621	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3622	* it from the APIC device.
3623	*
3624	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3625	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3626	* do that work when we execute nested-guest code esp. since all the required information
3627	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3628	* the virtual interrupt controller.
3629	*
3630	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3631	*/
3632	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3633	&& !pVCpu->hm.s.fSingleInstruction)
3634	{
3635	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3636	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3637	if ( fGif
3638	&& !fBlockInt
3639	&& !fIntShadow)
3640	{
3641	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3642	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3643	{
3644	Log4(("Intercepting INTR -> #VMEXIT\n"));
3645	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3646	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3647	}
3648	#endif
3649	uint8_t u8Interrupt;
3650	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3651	if (RT_SUCCESS(rc))
3652	{
3653	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3654	SVMEVENT Event;
3655	Event.u = 0;
3656	Event.n.u1Valid = 1;
3657	Event.n.u8Vector = u8Interrupt;
3658	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3659	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3660	}
3661	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3662	{
3663	/*
3664	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3665	* updated eventually when the TPR is written by the guest.
3666	*/
3667	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3668	}
3669	else
3670	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3671	}
3672	else if (!fGif)
3673	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3674	else if (!pSvmTransient->fIsNestedGuest)
3675	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3676	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3677	}
3678
3679	return VINF_SUCCESS;
3680	}
3681
3682
3683	/**
3684	* Injects any pending events into the guest (or nested-guest).
3685	*
3686	* @param pVCpu The cross context virtual CPU structure.
3687	* @param pVmcb Pointer to the VM control block.
3688	*
3689	* @remarks Must only be called when we are guaranteed to enter
3690	* hardware-assisted SVM execution and not return to ring-3
3691	* prematurely.
3692	*/
3693	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3694	{
3695	Assert(!TRPMHasTrap(pVCpu));
3696	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3697
3698	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3699	#ifdef VBOX_STRICT
3700	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3701	bool const fGif = CPUMGetGuestGif(pCtx);
3702	bool fAllowInt = fGif;
3703	if (fGif)
3704	{
3705	/*
3706	* For nested-guests we have no way to determine if we're injecting a physical or
3707	* virtual interrupt at this point. Hence the partial verification below.
3708	*/
3709	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3710	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3711	else
3712	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3713	}
3714	#endif
3715
3716	if (pVCpu->hm.s.Event.fPending)
3717	{
3718	SVMEVENT Event;
3719	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3720	Assert(Event.n.u1Valid);
3721
3722	/*
3723	* Validate event injection pre-conditions.
3724	*/
3725	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3726	{
3727	Assert(fAllowInt);
3728	Assert(!fIntShadow);
3729	}
3730	else if (Event.n.u3Type == SVM_EVENT_NMI)
3731	{
3732	Assert(fGif);
3733	Assert(!fIntShadow);
3734	}
3735
3736	/*
3737	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3738	* do this only when we are surely going to inject the NMI as otherwise if we return
3739	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3740	* SVM R0.
3741	*
3742	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3743	* which will set the VMCS field after actually delivering the NMI which we read on
3744	* VM-exit to determine the state.
3745	*/
3746	if ( Event.n.u3Type == SVM_EVENT_NMI
3747	&& Event.n.u8Vector == X86_XCPT_NMI)
3748	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3749
3750	/*
3751	* Inject it (update VMCB for injection by the hardware).
3752	*/
3753	Log4(("Injecting pending HM event\n"));
3754	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3755	pVCpu->hm.s.Event.fPending = false;
3756
3757	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3758	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3759	else
3760	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3761	}
3762	else
3763	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3764
3765	/*
3766	* We could have injected an NMI through IEM and continue guest execution using
3767	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3768	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3769	*/
3770	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3771	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3772
3773	/*
3774	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3775	*
3776	* For nested-guests: We need to update it too for the scenario where IEM executes
3777	* the nested-guest but execution later continues here with an interrupt shadow active.
3778	*/
3779	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3780	}
3781
3782
3783	/**
3784	* Reports world-switch error and dumps some useful debug info.
3785	*
3786	* @param pVCpu The cross context virtual CPU structure.
3787	* @param rcVMRun The return code from VMRUN (or
3788	* VERR_SVM_INVALID_GUEST_STATE for invalid
3789	* guest-state).
3790	*/
3791	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3792	{
3793	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3794	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3795	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3796
3797	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3798	{
3799	#ifdef VBOX_STRICT
3800	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3801	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3802	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3803	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3804	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3805	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3806	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3807	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3808	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3809	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3810	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3811	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3812
3813	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3814	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3815	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3816
3817	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3818	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3819	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3820	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3821	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3822	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3823	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3824	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3825	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3826	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3827	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3828	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3829
3830	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3831	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3832	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3833	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3834	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3835	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3836	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3837	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3838	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3839	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3840	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3841	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3842	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3843	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3844	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3845	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3846	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3847	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3848	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3849	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3850
3851	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3852
3853	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3854	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3855
3856	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3857	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3858	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3859	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3860	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3861	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3862	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3863	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3864	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3865	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3866	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3867	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3868	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3869	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3870	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3871	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3872	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3873	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3874	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3875	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3876
3877	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3878	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3879
3880	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3881	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3882	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3883	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3884
3885	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3886	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3887
3888	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3889	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3890	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3891	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3892
3893	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3894	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3895	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3896	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3897	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3898	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3899	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3900
3901	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3902	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3903	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3904	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3905
3906	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3907	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3908	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3909
3910	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3911	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3912	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3913	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3914	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3915	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3916	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3917	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3918	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3919	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3920	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3921	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3922
3923	NOREF(pVmcb);
3924	#endif /* VBOX_STRICT */
3925	}
3926	else
3927	Log4Func(("rcVMRun=%d\n", rcVMRun));
3928	}
3929
3930
3931	/**
3932	* Check per-VM and per-VCPU force flag actions that require us to go back to
3933	* ring-3 for one reason or another.
3934	*
3935	* @returns Strict VBox status code (information status code included).
3936	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3937	* ring-3.
3938	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3939	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3940	* interrupts)
3941	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3942	* all EMTs to be in ring-3.
3943	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3944	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3945	* to the EM loop.
3946	*
3947	* @param pVCpu The cross context virtual CPU structure.
3948	*/
3949	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3950	{
3951	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3952
3953	/* Could happen as a result of longjump. */
3954	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3955	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3956
3957	/* Update pending interrupts into the APIC's IRR. */
3958	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3959	PDMApicUpdatePendingInterrupts(pVCpu);
3960
3961	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3962	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3963	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3964	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3965	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3966	{
3967	/* Pending PGM C3 sync. */
3968	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3969	{
3970	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3971	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3972	if (rc != VINF_SUCCESS)
3973	{
3974	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3975	return rc;
3976	}
3977	}
3978
3979	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3980	/* -XXX- what was that about single stepping? */
3981	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3982	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3983	{
3984	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3985	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3986	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3987	return rc;
3988	}
3989
3990	/* Pending VM request packets, such as hardware interrupts. */
3991	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3992	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3993	{
3994	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3995	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3996	return VINF_EM_PENDING_REQUEST;
3997	}
3998
3999	/* Pending PGM pool flushes. */
4000	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4001	{
4002	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4003	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4004	return VINF_PGM_POOL_FLUSH_PENDING;
4005	}
4006
4007	/* Pending DMA requests. */
4008	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4009	{
4010	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4011	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4012	return VINF_EM_RAW_TO_R3;
4013	}
4014	}
4015
4016	return VINF_SUCCESS;
4017	}
4018
4019
4020	/**
4021	* Does the preparations before executing guest code in AMD-V.
4022	*
4023	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4024	* recompiler. We must be cautious what we do here regarding committing
4025	* guest-state information into the VMCB assuming we assuredly execute the guest
4026	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4027	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4028	* that the recompiler can (and should) use them when it resumes guest
4029	* execution. Otherwise such operations must be done when we can no longer
4030	* exit to ring-3.
4031	*
4032	* @returns Strict VBox status code (informational status codes included).
4033	* @retval VINF_SUCCESS if we can proceed with running the guest.
4034	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4035	*
4036	* @param pVCpu The cross context virtual CPU structure.
4037	* @param pSvmTransient Pointer to the SVM transient structure.
4038	*/
4039	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4040	{
4041	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4042
4043	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4044	if (pSvmTransient->fIsNestedGuest)
4045	{
4046	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4047	return VINF_EM_RESCHEDULE_REM;
4048	}
4049	#endif
4050
4051	/* Check force flag actions that might require us to go back to ring-3. */
4052	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4053	if (rc != VINF_SUCCESS)
4054	return rc;
4055
4056	if (TRPMHasTrap(pVCpu))
4057	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4058	else if (!pVCpu->hm.s.Event.fPending)
4059	{
4060	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4061	if ( rc != VINF_SUCCESS
4062	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4063	{
4064	/* If a nested-guest VM-exit occurred, bail. */
4065	if (pSvmTransient->fIsNestedGuest)
4066	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4067	return rc;
4068	}
4069	}
4070
4071	/*
4072	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4073	* Just do it in software, see @bugref{8411}.
4074	* NB: If we could continue a task switch exit we wouldn't need to do this.
4075	*/
4076	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4077	if (RT_UNLIKELY( !g_fHmSvmFeatures
4078	&& pVCpu->hm.s.Event.fPending
4079	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4080	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4081
4082	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4083	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4084	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4085	#endif
4086
4087	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4088	/*
4089	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4090	*/
4091	if (pSvmTransient->fIsNestedGuest)
4092	hmR0SvmSetupVmcbNested(pVCpu);
4093	#endif
4094
4095	/*
4096	* Export the guest state bits that are not shared with the host in any way as we can
4097	* longjmp or get preempted in the midst of exporting some of the state.
4098	*/
4099	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4100	AssertRCReturn(rc, rc);
4101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4102
4103	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4104	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4105
4106	/*
4107	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4108	* world-switch so we can update it on the way back if the guest changed the TPR.
4109	*/
4110	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4111	{
4112	Assert(!pSvmTransient->fIsNestedGuest);
4113	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4114	if (pVM->hm.s.fTprPatchingActive)
4115	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4116	else
4117	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4118	}
4119
4120	/*
4121	* No longjmps to ring-3 from this point on!!!
4122	*
4123	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4124	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4125	*/
4126	VMMRZCallRing3Disable(pVCpu);
4127
4128	/*
4129	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4130	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4131	* preemption disabled for a while. Since this is purly to aid the
4132	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4133	* disable interrupt on NT.
4134	*
4135	* We need to check for force-flags that could've possible been altered since we last
4136	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4137	* see @bugref{6398}).
4138	*
4139	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4140	* to ring-3 before executing guest code.
4141	*/
4142	pSvmTransient->fEFlags = ASMIntDisableFlags();
4143	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4144	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4145	{
4146	ASMSetFlags(pSvmTransient->fEFlags);
4147	VMMRZCallRing3Enable(pVCpu);
4148	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4149	return VINF_EM_RAW_TO_R3;
4150	}
4151	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4152	{
4153	ASMSetFlags(pSvmTransient->fEFlags);
4154	VMMRZCallRing3Enable(pVCpu);
4155	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4156	return VINF_EM_RAW_INTERRUPT;
4157	}
4158
4159	return VINF_SUCCESS;
4160	}
4161
4162
4163	/**
4164	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4165	* doing so.
4166	*
4167	* This means there is no backing out to ring-3 or anywhere else at this point.
4168	*
4169	* @param pVCpu The cross context virtual CPU structure.
4170	* @param pSvmTransient Pointer to the SVM transient structure.
4171	*
4172	* @remarks Called with preemption disabled.
4173	* @remarks No-long-jump zone!!!
4174	*/
4175	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4176	{
4177	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4179
4180	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4181	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4182
4183	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4184	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4185
4186	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4187
4188	if (!CPUMIsGuestFPUStateActive(pVCpu))
4189	{
4190	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4191	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4192	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4193	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4194	}
4195
4196	/* Load the state shared between host and guest (FPU, debug). */
4197	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4198	hmR0SvmExportSharedState(pVCpu, pVmcb);
4199
4200	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4201	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4202
4203	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4204	RTCPUID const idHostCpu = pHostCpu->idCpu;
4205	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4206
4207	/* Setup TSC offsetting. */
4208	if ( pSvmTransient->fUpdateTscOffsetting
4209	\|\| fMigratedHostCpu)
4210	{
4211	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4212	pSvmTransient->fUpdateTscOffsetting = false;
4213	}
4214
4215	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4216	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4217	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4218	else
4219	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4220
4221	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4222	if (fMigratedHostCpu)
4223	pVmcb->ctrl.u32VmcbCleanBits = 0;
4224
4225	/* Store status of the shared guest-host state at the time of VMRUN. */
4226	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4227	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4228
4229	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4230	uint8_t *pbMsrBitmap;
4231	if (!pSvmTransient->fIsNestedGuest)
4232	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4233	else
4234	{
4235	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4236	* MSRPM and only perform this if it changed also use EVEX.POR when it
4237	* does. */
4238	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4239
4240	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4241	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4242	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4243	}
4244	#else
4245	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4246	#endif
4247
4248	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4249	/* Flush the appropriate tagged-TLB entries. */
4250	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4251	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4252
4253	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4254
4255	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4256	to start executing. */
4257
4258	/*
4259	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4260	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4261	*
4262	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4263	*/
4264	if ( g_CpumHostFeatures.s.fRdTscP
4265	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4266	{
4267	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4268	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4269	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4270	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4271	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4272	pSvmTransient->fRestoreTscAuxMsr = true;
4273	}
4274	else
4275	{
4276	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4277	pSvmTransient->fRestoreTscAuxMsr = false;
4278	}
4279	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4280
4281	/*
4282	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4283	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4284	* from the VMCB.
4285	*/
4286	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4287	if (!fSupportsVmcbCleanBits)
4288	pVmcb->ctrl.u32VmcbCleanBits = 0;
4289	}
4290
4291
4292	/**
4293	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4294	*
4295	* @returns VBox strict status code.
4296	* @param pVCpu The cross context virtual CPU structure.
4297	* @param HCPhysVmcb The host physical address of the VMCB.
4298	*
4299	* @remarks No-long-jump zone!!!
4300	*/
4301	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4302	{
4303	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4304	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4305	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4306	}
4307
4308
4309	/**
4310	* Performs some essential restoration of state after running guest (or
4311	* nested-guest) code in AMD-V.
4312	*
4313	* @param pVCpu The cross context virtual CPU structure.
4314	* @param pSvmTransient Pointer to the SVM transient structure.
4315	* @param rcVMRun Return code of VMRUN.
4316	*
4317	* @remarks Called with interrupts disabled.
4318	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4319	* unconditionally when it is safe to do so.
4320	*/
4321	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4322	{
4323	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4324
4325	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4326	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4327
4328	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4329	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4330
4331	/* TSC read must be done early for maximum accuracy. */
4332	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4333	{
4334	if (!pSvmTransient->fIsNestedGuest)
4335	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4336	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4337	else
4338	{
4339	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4340	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4341	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4342	}
4343	#endif
4344	}
4345
4346	if (pSvmTransient->fRestoreTscAuxMsr)
4347	{
4348	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4349	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4350	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4351	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4352	}
4353
4354	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4355	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4356	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4357	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4358
4359	Assert(!(ASMGetFlags() & X86_EFL_IF));
4360	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4361	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4362
4363	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4364	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4365	{
4366	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4367	return;
4368	}
4369
4370	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4371	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4372	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4373	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4374
4375	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4376	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4377	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4378	#else
4379	/*
4380	* Always import the following:
4381	*
4382	* - RIP for exit optimizations and evaluating event injection on re-entry.
4383	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4384	* state on preemption.
4385	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4386	* - CS for exit optimizations.
4387	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4388	* assembly switcher code.
4389	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4390	*/
4391	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4392	\| CPUMCTX_EXTRN_RFLAGS
4393	\| CPUMCTX_EXTRN_RAX
4394	\| CPUMCTX_EXTRN_RSP
4395	\| CPUMCTX_EXTRN_CS
4396	\| CPUMCTX_EXTRN_HWVIRT
4397	\| CPUMCTX_EXTRN_INHIBIT_INT
4398	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4399	\| HMSVM_CPUMCTX_SHARED_STATE);
4400	#endif
4401
4402	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4403	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4404	{
4405	Assert(!pSvmTransient->fIsNestedGuest);
4406	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4407	if ( pVM->hm.s.fTprPatchingActive
4408	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4409	{
4410	int rc = PDMApicSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4411	AssertRC(rc);
4412	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4413	}
4414	/* Sync TPR when we aren't intercepting CR8 writes. */
4415	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4416	{
4417	int rc = PDMApicSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4418	AssertRC(rc);
4419	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4420	}
4421	}
4422
4423	#ifdef DEBUG_ramshankar
4424	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4425	{
4426	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4427	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4428	0 /* uVerbose */);
4429	}
4430	#endif
4431
4432	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4433	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4434	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4435	}
4436
4437
4438	/**
4439	* Runs the guest code using AMD-V.
4440	*
4441	* @returns Strict VBox status code.
4442	* @param pVCpu The cross context virtual CPU structure.
4443	* @param pcLoops Pointer to the number of executed loops.
4444	*/
4445	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4446	{
4447	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4448	Assert(pcLoops);
4449	Assert(*pcLoops <= cMaxResumeLoops);
4450
4451	SVMTRANSIENT SvmTransient;
4452	RT_ZERO(SvmTransient);
4453	SvmTransient.fUpdateTscOffsetting = true;
4454	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4455
4456	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4457	for (;;)
4458	{
4459	Assert(!HMR0SuspendPending());
4460	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4461
4462	/* Preparatory work for running nested-guest code, this may force us to return to
4463	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4464	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4465	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4466	if (rc != VINF_SUCCESS)
4467	break;
4468
4469	/*
4470	* No longjmps to ring-3 from this point on!!!
4471	*
4472	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4473	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4474	*/
4475	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4476	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4477
4478	/* Restore any residual host-state and save any bits shared between host and guest
4479	into the guest-CPU state. Re-enables interrupts! */
4480	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4481
4482	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4483	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4484	{
4485	if (rc == VINF_SUCCESS)
4486	rc = VERR_SVM_INVALID_GUEST_STATE;
4487	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4488	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4489	break;
4490	}
4491
4492	/* Handle the #VMEXIT. */
4493	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4494	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4495	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4496	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4497	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4498	if (rc != VINF_SUCCESS)
4499	break;
4500	if (++(*pcLoops) >= cMaxResumeLoops)
4501	{
4502	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4503	rc = VINF_EM_RAW_INTERRUPT;
4504	break;
4505	}
4506	}
4507
4508	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4509	return rc;
4510	}
4511
4512
4513	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4514	/**
4515	* Runs the nested-guest code using AMD-V.
4516	*
4517	* @returns Strict VBox status code.
4518	* @param pVCpu The cross context virtual CPU structure.
4519	* @param pcLoops Pointer to the number of executed loops. If we're switching
4520	* from the guest-code execution loop to this nested-guest
4521	* execution loop pass the remainder value, else pass 0.
4522	*/
4523	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4524	{
4525	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4526	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4527	Assert(pcLoops);
4528	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4529	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4530	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4531
4532	SVMTRANSIENT SvmTransient;
4533	RT_ZERO(SvmTransient);
4534	SvmTransient.fUpdateTscOffsetting = true;
4535	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4536	SvmTransient.fIsNestedGuest = true;
4537
4538	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4539	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4540
4541	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4542	for (;;)
4543	{
4544	Assert(!HMR0SuspendPending());
4545	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4546
4547	/* Preparatory work for running nested-guest code, this may force us to return to
4548	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4549	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4550	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4551	if ( rc != VINF_SUCCESS
4552	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4553	break;
4554
4555	/*
4556	* No longjmps to ring-3 from this point on!!!
4557	*
4558	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4559	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4560	*/
4561	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4562
4563	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4564
4565	/* Restore any residual host-state and save any bits shared between host and guest
4566	into the guest-CPU state. Re-enables interrupts! */
4567	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4568
4569	if (RT_LIKELY( rc == VINF_SUCCESS
4570	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4571	{ /* extremely likely */ }
4572	else
4573	{
4574	/* VMRUN failed, shouldn't really happen, Guru. */
4575	if (rc != VINF_SUCCESS)
4576	break;
4577
4578	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4579	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4580	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4581	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4582	break;
4583	}
4584
4585	/* Handle the #VMEXIT. */
4586	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4587	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4588	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4589	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4590	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4591	if (rc == VINF_SUCCESS)
4592	{
4593	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4594	{
4595	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4596	rc = VINF_SVM_VMEXIT;
4597	}
4598	else
4599	{
4600	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4601	continue;
4602	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4603	rc = VINF_EM_RAW_INTERRUPT;
4604	}
4605	}
4606	else
4607	Assert(rc != VINF_SVM_VMEXIT);
4608	break;
4609	/** @todo NSTSVM: handle single-stepping. */
4610	}
4611
4612	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4613	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4614
4615	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4616	return rc;
4617	}
4618	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4619
4620
4621	/**
4622	* Checks if any expensive dtrace probes are enabled and we should go to the
4623	* debug loop.
4624	*
4625	* @returns true if we should use debug loop, false if not.
4626	*/
4627	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4628	{
4629	/* It's probably faster to OR the raw 32-bit counter variables together.
4630	Since the variables are in an array and the probes are next to one
4631	another (more or less), we have good locality. So, better read
4632	eight-nine cache lines ever time and only have one conditional, than
4633	128+ conditionals, right? */
4634	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4635	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4651	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4652	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4653	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4654	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4655	) != 0
4656	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4680	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4682	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4683	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4684	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4685	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4686	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4687	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4692	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4693	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4694	) != 0
4695	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4720	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4722	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4723	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4724	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4725	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4726	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4727	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4732	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4733	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4734	) != 0;
4735	}
4736
4737
4738	/**
4739	* Runs the guest code using AMD-V.
4740	*
4741	* @returns Strict VBox status code.
4742	* @param pVCpu The cross context virtual CPU structure.
4743	*/
4744	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4745	{
4746	AssertPtr(pVCpu);
4747	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4748	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4749	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4750	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4751
4752	uint32_t cLoops = 0;
4753	VBOXSTRICTRC rc;
4754	for (;;)
4755	{
4756	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4757	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4758	#else
4759	NOREF(pCtx);
4760	bool const fInNestedGuestMode = false;
4761	#endif
4762	if (!fInNestedGuestMode)
4763	{
4764	if ( !pVCpu->hm.s.fUseDebugLoop
4765	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4766	&& !DBGFIsStepping(pVCpu)
4767	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledSwBreakpoints)
4768	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4769	else
4770	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4771	}
4772	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4773	else
4774	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4775
4776	if (rc == VINF_SVM_VMRUN)
4777	{
4778	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4779	continue;
4780	}
4781	if (rc == VINF_SVM_VMEXIT)
4782	{
4783	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4784	continue;
4785	}
4786	#endif
4787	break;
4788	}
4789
4790	/* Fixup error codes. */
4791	if (rc == VERR_EM_INTERPRETER)
4792	rc = VINF_EM_RAW_EMULATE_INSTR;
4793	else if (rc == VINF_EM_RESET)
4794	rc = VINF_EM_TRIPLE_FAULT;
4795
4796	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4797	rc = hmR0SvmExitToRing3(pVCpu, rc);
4798	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4799	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4800	return rc;
4801	}
4802
4803	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4804
4805	/**
4806	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4807	*
4808	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4809	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4810	*/
4811	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4812	{
4813	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4814	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4815	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4816	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4817	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4818	const bool fRep = pIoExitInfo->n.u1Rep;
4819	const bool fStrIo = pIoExitInfo->n.u1Str;
4820
4821	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4822	NULL /* pIoExitInfo */);
4823	}
4824
4825
4826	/**
4827	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4828	* SVM_EXIT_INVALID).
4829	*
4830	* @returns VBox status code (informational status codes included).
4831	* @param pVCpu The cross context virtual CPU structure.
4832	* @param pSvmTransient Pointer to the SVM transient structure.
4833	*/
4834	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4835	{
4836	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4837	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4838	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4839
4840	/*
4841	* We import the complete state here because we use separate VMCBs for the guest and the
4842	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4843	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4844	*/
4845	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4846	do { \
4847	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4848	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4849	} while (0)
4850
4851	/*
4852	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4853	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4854	*/
4855	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4856	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4857	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4858	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4859	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4860	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4861
4862	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4863	switch (uExitCode)
4864	{
4865	case SVM_EXIT_CPUID:
4866	{
4867	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4868	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4869	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4870	}
4871
4872	case SVM_EXIT_RDTSC:
4873	{
4874	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4875	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4876	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4877	}
4878
4879	case SVM_EXIT_RDTSCP:
4880	{
4881	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4882	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4883	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4884	}
4885
4886	case SVM_EXIT_MONITOR:
4887	{
4888	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4889	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4890	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4891	}
4892
4893	case SVM_EXIT_MWAIT:
4894	{
4895	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4896	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4897	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4898	}
4899
4900	case SVM_EXIT_HLT:
4901	{
4902	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4903	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4904	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4905	}
4906
4907	case SVM_EXIT_MSR:
4908	{
4909	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4910	{
4911	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4912	uint16_t offMsrpm;
4913	uint8_t uMsrpmBit;
4914	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4915	if (RT_SUCCESS(rc))
4916	{
4917	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4918	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4919
4920	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4921	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4922	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4923
4924	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4925	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4926	{
4927	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4928	}
4929	}
4930	else
4931	{
4932	/*
4933	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4934	* See AMD-V spec. "15.11 MSR Intercepts".
4935	*/
4936	Assert(rc == VERR_OUT_OF_RANGE);
4937	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4938	}
4939	}
4940	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4941	}
4942
4943	case SVM_EXIT_IOIO:
4944	{
4945	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4946	{
4947	SVMIOIOEXITINFO IoExitInfo;
4948	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4949	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4950	if (fIntercept)
4951	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4952	}
4953	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4954	}
4955
4956	case SVM_EXIT_XCPT_PF:
4957	{
4958	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4959	if (pVM->hmr0.s.fNestedPaging)
4960	{
4961	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4962	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4963
4964	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4965	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4966	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4967
4968	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4969	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4970	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4971	return VINF_SUCCESS;
4972	}
4973	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4974	}
4975
4976	case SVM_EXIT_XCPT_UD:
4977	{
4978	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4979	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4980	hmR0SvmSetPendingXcptUD(pVCpu);
4981	return VINF_SUCCESS;
4982	}
4983
4984	case SVM_EXIT_XCPT_MF:
4985	{
4986	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4987	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4988	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4989	}
4990
4991	case SVM_EXIT_XCPT_DB:
4992	{
4993	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4994	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4995	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4996	}
4997
4998	case SVM_EXIT_XCPT_AC:
4999	{
5000	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5001	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5002	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5003	}
5004
5005	case SVM_EXIT_XCPT_BP:
5006	{
5007	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5008	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5009	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5010	}
5011
5012	case SVM_EXIT_READ_CR0:
5013	case SVM_EXIT_READ_CR3:
5014	case SVM_EXIT_READ_CR4:
5015	{
5016	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5017	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5018	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5019	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5020	}
5021
5022	case SVM_EXIT_CR0_SEL_WRITE:
5023	{
5024	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5025	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5026	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5027	}
5028
5029	case SVM_EXIT_WRITE_CR0:
5030	case SVM_EXIT_WRITE_CR3:
5031	case SVM_EXIT_WRITE_CR4:
5032	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5033	{
5034	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5035	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5036
5037	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5038	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5039	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5040	}
5041
5042	case SVM_EXIT_PAUSE:
5043	{
5044	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5045	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5046	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5047	}
5048
5049	case SVM_EXIT_VINTR:
5050	{
5051	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5052	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5053	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5054	}
5055
5056	case SVM_EXIT_INTR:
5057	case SVM_EXIT_NMI:
5058	case SVM_EXIT_SMI:
5059	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5060	{
5061	/*
5062	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5063	*
5064	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5065	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5066	* nested-guest #VMEXIT.
5067	*
5068	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5069	* while trying to inject interrupts, see comment at the top of this function.
5070	*/
5071	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5072	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5073	}
5074
5075	case SVM_EXIT_FERR_FREEZE:
5076	{
5077	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5078	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5079	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5080	}
5081
5082	case SVM_EXIT_INVLPG:
5083	{
5084	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5085	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5086	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5087	}
5088
5089	case SVM_EXIT_WBINVD:
5090	{
5091	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5092	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5093	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5094	}
5095
5096	case SVM_EXIT_INVD:
5097	{
5098	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5099	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5100	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5101	}
5102
5103	case SVM_EXIT_RDPMC:
5104	{
5105	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5106	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5107	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5108	}
5109
5110	default:
5111	{
5112	switch (uExitCode)
5113	{
5114	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5115	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5116	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5117	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5118	{
5119	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5120	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5121	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5122	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5123	}
5124
5125	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5126	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5127	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5128	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5129	{
5130	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5131	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5132	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5133	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5134	}
5135
5136	case SVM_EXIT_XCPT_DE:
5137	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5138	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5139	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5140	case SVM_EXIT_XCPT_OF:
5141	case SVM_EXIT_XCPT_BR:
5142	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5143	case SVM_EXIT_XCPT_NM:
5144	case SVM_EXIT_XCPT_DF:
5145	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5146	case SVM_EXIT_XCPT_TS:
5147	case SVM_EXIT_XCPT_NP:
5148	case SVM_EXIT_XCPT_SS:
5149	case SVM_EXIT_XCPT_GP:
5150	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5151	case SVM_EXIT_XCPT_15: /* Reserved. */
5152	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5153	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5154	case SVM_EXIT_XCPT_MC:
5155	case SVM_EXIT_XCPT_XF:
5156	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5157	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5158	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5159	{
5160	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5161	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5162	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5163	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5164	}
5165
5166	case SVM_EXIT_XSETBV:
5167	{
5168	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5169	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5170	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5171	}
5172
5173	case SVM_EXIT_TASK_SWITCH:
5174	{
5175	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5176	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5177	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5178	}
5179
5180	case SVM_EXIT_IRET:
5181	{
5182	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5183	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5184	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5185	}
5186
5187	case SVM_EXIT_SHUTDOWN:
5188	{
5189	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5190	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5191	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5192	}
5193
5194	case SVM_EXIT_VMMCALL:
5195	{
5196	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5197	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5198	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5199	}
5200
5201	case SVM_EXIT_CLGI:
5202	{
5203	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5204	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5205	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5206	}
5207
5208	case SVM_EXIT_STGI:
5209	{
5210	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5211	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5212	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5213	}
5214
5215	case SVM_EXIT_VMLOAD:
5216	{
5217	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5218	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5219	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5220	}
5221
5222	case SVM_EXIT_VMSAVE:
5223	{
5224	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5225	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5226	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5227	}
5228
5229	case SVM_EXIT_INVLPGA:
5230	{
5231	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5232	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5233	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5234	}
5235
5236	case SVM_EXIT_VMRUN:
5237	{
5238	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5239	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5240	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5241	}
5242
5243	case SVM_EXIT_RSM:
5244	{
5245	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5246	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5247	hmR0SvmSetPendingXcptUD(pVCpu);
5248	return VINF_SUCCESS;
5249	}
5250
5251	case SVM_EXIT_SKINIT:
5252	{
5253	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5254	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5255	hmR0SvmSetPendingXcptUD(pVCpu);
5256	return VINF_SUCCESS;
5257	}
5258
5259	case SVM_EXIT_NPF:
5260	{
5261	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5262	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5263	}
5264
5265	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5266	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5267
5268	default:
5269	{
5270	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5271	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5272	return VERR_SVM_UNKNOWN_EXIT;
5273	}
5274	}
5275	}
5276	}
5277	/* not reached */
5278
5279	# undef NST_GST_VMEXIT_CALL_RET
5280	}
5281
5282	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5283
5284	/** @def VMEXIT_CALL_RET
5285	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5286	*/
5287	#ifdef DEBUG_ramshankar
5288	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5289	do { \
5290	if ((a_fDbg) == 1) \
5291	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5292	int rc = a_CallExpr; \
5293	if ((a_fDbg) == 1) \
5294	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5295	return rc; \
5296	} while (0)
5297	#else
5298	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5299	#endif
5300
5301	/**
5302	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5303	*
5304	* @returns Strict VBox status code (informational status codes included).
5305	* @param pVCpu The cross context virtual CPU structure.
5306	* @param pSvmTransient Pointer to the SVM transient structure.
5307	*/
5308	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5309	{
5310	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5311	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5312
5313	/*
5314	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5315	* for most guests under normal workloads (for some definition of "normal").
5316	*/
5317	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5318	switch (uExitCode)
5319	{
5320	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5321	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5322	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5323	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5324	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5325	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5326	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5327	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5328	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5329	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5330
5331	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5332	case SVM_EXIT_INTR:
5333	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5334
5335	case SVM_EXIT_READ_CR0:
5336	case SVM_EXIT_READ_CR3:
5337	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5338
5339	case SVM_EXIT_CR0_SEL_WRITE:
5340	case SVM_EXIT_WRITE_CR0:
5341	case SVM_EXIT_WRITE_CR3:
5342	case SVM_EXIT_WRITE_CR4:
5343	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5344
5345	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5346	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5347	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5348	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5349	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5350	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5351	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5352	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5353	case SVM_EXIT_XCPT_DE: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDE(pVCpu, pSvmTransient));
5354	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5355	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5356	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5357	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5358	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5359	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5360	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5361	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5362
5363	default:
5364	{
5365	switch (pSvmTransient->u64ExitCode)
5366	{
5367	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5368	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5369	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5370	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5371	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5372
5373	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5374	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5375	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5376	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5377	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5378
5379	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5380	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5381
5382	case SVM_EXIT_SMI:
5383	case SVM_EXIT_INIT:
5384	{
5385	/*
5386	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5387	* If it ever does, we want to know about it so log the exit code and bail.
5388	*/
5389	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5390	}
5391
5392	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5393	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5394	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5395	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5396	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5397	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5398	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5399	#else
5400	case SVM_EXIT_CLGI:
5401	case SVM_EXIT_STGI:
5402	case SVM_EXIT_VMLOAD:
5403	case SVM_EXIT_VMSAVE:
5404	case SVM_EXIT_INVLPGA:
5405	case SVM_EXIT_VMRUN:
5406	#endif
5407	case SVM_EXIT_RSM:
5408	case SVM_EXIT_SKINIT:
5409	{
5410	hmR0SvmSetPendingXcptUD(pVCpu);
5411	return VINF_SUCCESS;
5412	}
5413
5414	/*
5415	* The remaining should only be possible when debugging or dtracing.
5416	*/
5417	case SVM_EXIT_XCPT_DE:
5418	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5419	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5420	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5421	case SVM_EXIT_XCPT_OF:
5422	case SVM_EXIT_XCPT_BR:
5423	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5424	case SVM_EXIT_XCPT_NM:
5425	case SVM_EXIT_XCPT_DF:
5426	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5427	case SVM_EXIT_XCPT_TS:
5428	case SVM_EXIT_XCPT_NP:
5429	case SVM_EXIT_XCPT_SS:
5430	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5431	/* SVM_EXIT_XCPT_PF: */
5432	case SVM_EXIT_XCPT_15: /* Reserved. */
5433	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5434	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5435	case SVM_EXIT_XCPT_MC:
5436	case SVM_EXIT_XCPT_XF:
5437	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5438	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5439	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5440	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5441
5442	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5443	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5444	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5445
5446	default:
5447	{
5448	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5449	pVCpu->hm.s.u32HMError = uExitCode;
5450	return VERR_SVM_UNKNOWN_EXIT;
5451	}
5452	}
5453	}
5454	}
5455	/* not reached */
5456	}
5457
5458
5459	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5460	*
5461	* The following few functions and associated structure contains the bloat
5462	* necessary for providing detailed debug events and dtrace probes as well as
5463	* reliable host side single stepping. This works on the principle of
5464	* "subclassing" the normal execution loop and workers. We replace the loop
5465	* method completely and override selected helpers to add necessary adjustments
5466	* to their core operation.
5467	*
5468	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5469	* any performance for debug and analysis features.
5470	*
5471	* @{
5472	*/
5473
5474	/**
5475	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5476	* the debug run loop.
5477	*/
5478	typedef struct SVMRUNDBGSTATE
5479	{
5480	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5481	uint64_t bmInterceptInitial;
5482	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5483	uint32_t bmXcptInitial;
5484	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5485	uint16_t bmInterceptRdCRxInitial;
5486	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5487	uint16_t bmInterceptWrCRxInitial;
5488	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5489	uint16_t bmInterceptRdDRxInitial;
5490	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5491	uint16_t bmInterceptWrDRxInitial;
5492
5493	/** Whether we've actually modified the intercept control qword. */
5494	bool fModifiedInterceptCtrl : 1;
5495	/** Whether we've actually modified the exception bitmap. */
5496	bool fModifiedXcptBitmap : 1;
5497	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5498	bool fModifiedInterceptRdCRx : 1;
5499	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5500	bool fModifiedInterceptWrCRx : 1;
5501	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5502	bool fModifiedInterceptRdDRx : 1;
5503	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5504	bool fModifiedInterceptWrDRx : 1;
5505
5506	/** The CS we started executing with. */
5507	uint16_t uCsStart;
5508	/** The RIP we started executing at. This is for detecting that we stepped. */
5509	uint64_t uRipStart;
5510
5511	/** The sequence number of the Dtrace provider settings the state was
5512	* configured against. */
5513	uint32_t uDtraceSettingsSeqNo;
5514	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5515	uint32_t bmXcptExtra;
5516	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5517	uint64_t bmInterceptExtra;
5518	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5519	uint16_t bmInterceptRdCRxExtra;
5520	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5521	uint16_t bmInterceptWrCRxExtra;
5522	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5523	uint16_t bmInterceptRdDRxExtra;
5524	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5525	uint16_t bmInterceptWrDRxExtra;
5526	/** VM-exits to check (one bit per VM-exit). */
5527	uint32_t bmExitsToCheck[33];
5528	} SVMRUNDBGSTATE;
5529	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5530	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5531
5532
5533	/**
5534	* Initializes the SVMRUNDBGSTATE structure.
5535	*
5536	* @param pVCpu The cross context virtual CPU structure of the
5537	* calling EMT.
5538	* @param pSvmTransient The SVM-transient structure.
5539	* @param pDbgState The debug state to initialize.
5540	*/
5541	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5542	{
5543	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5544	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5545	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5546	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5547	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5548	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5549	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5550
5551	pDbgState->fModifiedInterceptCtrl = false;
5552	pDbgState->fModifiedXcptBitmap = false;
5553	pDbgState->fModifiedInterceptRdCRx = false;
5554	pDbgState->fModifiedInterceptWrCRx = false;
5555	pDbgState->fModifiedInterceptRdDRx = false;
5556	pDbgState->fModifiedInterceptWrDRx = false;
5557
5558	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5559	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5560
5561	/* We don't really need to zero these. */
5562	pDbgState->bmInterceptExtra = 0;
5563	pDbgState->bmXcptExtra = 0;
5564	pDbgState->bmInterceptRdCRxExtra = 0;
5565	pDbgState->bmInterceptWrCRxExtra = 0;
5566	pDbgState->bmInterceptRdDRxExtra = 0;
5567	pDbgState->bmInterceptWrDRxExtra = 0;
5568	}
5569
5570
5571	/**
5572	* Updates the VMCB fields with changes requested by @a pDbgState.
5573	*
5574	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5575	* immediately before executing guest code, i.e. when interrupts are disabled.
5576	* We don't check status codes here as we cannot easily assert or return in the
5577	* latter case.
5578	*
5579	* @param pSvmTransient The SVM-transient structure.
5580	* @param pDbgState The debug state.
5581	*/
5582	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5583	{
5584	/*
5585	* Ensure desired flags in VMCS control fields are set.
5586	*/
5587	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5588	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5589	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5590	{ \
5591	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5592	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5593	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5594	(a_fModified) = true; \
5595	} \
5596	} while (0)
5597	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5598	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5599	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5600	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5601	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5602	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5603	#undef ADD_EXTRA_INTERCEPTS
5604	}
5605
5606
5607	/**
5608	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5609	* for re-entry next time around.
5610	*
5611	* @param pSvmTransient The SVM-transient structure.
5612	* @param pDbgState The debug state.
5613	*/
5614	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5615	{
5616	/*
5617	* Restore VM-exit control settings as we may not reenter this function the
5618	* next time around.
5619	*/
5620	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5621
5622	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5623	if ((a_fModified)) \
5624	{ \
5625	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5626	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5627	} \
5628	} while (0)
5629	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5630	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5631	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5632	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5633	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5634	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5635	#undef RESTORE_INTERCEPTS
5636	}
5637
5638
5639	/**
5640	* Configures VM-exit controls for current DBGF and DTrace settings.
5641	*
5642	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5643	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5644	*
5645	* @param pVCpu The cross context virtual CPU structure.
5646	* @param pSvmTransient The SVM-transient structure. May update
5647	* fUpdatedTscOffsettingAndPreemptTimer.
5648	* @param pDbgState The debug state.
5649	*/
5650	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5651	{
5652	/*
5653	* Take down the dtrace serial number so we can spot changes.
5654	*/
5655	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5656	ASMCompilerBarrier();
5657
5658	/*
5659	* Clear data members that we'll be rebuilding here.
5660	*/
5661	pDbgState->bmXcptExtra = 0;
5662	pDbgState->bmInterceptExtra = 0;
5663	pDbgState->bmInterceptRdCRxExtra = 0;
5664	pDbgState->bmInterceptWrCRxExtra = 0;
5665	pDbgState->bmInterceptRdDRxExtra = 0;
5666	pDbgState->bmInterceptWrDRxExtra = 0;
5667	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5668	pDbgState->bmExitsToCheck[i] = 0;
5669
5670	/*
5671	* Software interrupts (INT XXh)
5672	*/
5673	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5674	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5675	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5676	{
5677	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5678	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5679	}
5680
5681	/*
5682	* INT3 breakpoints - triggered by #BP exceptions.
5683	*/
5684	if (pVM->dbgf.ro.cEnabledSwBreakpoints > 0)
5685	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5686
5687	/*
5688	* Exception bitmap and XCPT events+probes.
5689	*/
5690	#define SET_XCPT(a_iXcpt) do { \
5691	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5692	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5693	} while (0)
5694
5695	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5696	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5697	SET_XCPT(iXcpt);
5698
5699	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5700	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5701	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5702	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5703	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5704	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5705	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5706	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5707	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5708	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5709	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5710	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5711	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5712	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5713	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5714	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5715	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5716	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5717
5718	#undef SET_XCPT
5719
5720	/*
5721	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5722	*
5723	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5724	* So, when adding/changing/removing please don't forget to update it.
5725	*
5726	* Some of the macros are picking up local variables to save horizontal space,
5727	* (being able to see it in a table is the lesser evil here).
5728	*/
5729	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5730	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5731	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5732	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5733	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5734	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5735	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5736	} else do { } while (0)
5737	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5738	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5739	{ \
5740	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5741	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5742	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5743	} else do { } while (0)
5744
5745	/** @todo double check these */
5746	/** @todo Check what more AMD-V specific we can intercept. */
5747	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5748	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5749	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5750	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5751	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5752	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5753	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5754	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5755	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5756	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5757	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5758	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5759	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5760	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5761
5762	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5763	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5764	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5765	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5766	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5767	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5768	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5769	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5770	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5771	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5772	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5773	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5774	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5775	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5776	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5777	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5778
5779	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5780	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5781	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5782	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5783
5784	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5785	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5786	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5787	{
5788	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5789	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5790	}
5791
5792	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5793	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5794	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5795	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5796
5797	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5798	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5799	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5800	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5801
5802	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5804	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5806	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5808	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5809	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5810	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5811	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5812	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5813	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5814	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5816	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5818	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5819	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5820	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5821	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5822	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5823	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5824	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5825	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5826	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5827	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5828	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5829	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5830	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5832	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5833	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5834
5835	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5836	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5837
5838	#undef IS_EITHER_ENABLED
5839	#undef SET_ONLY_XBM_IF_EITHER_EN
5840	#undef SET_INCP_XBM_IF_EITHER_EN
5841
5842	/*
5843	* Sanitize the control stuff.
5844	*/
5845	/** @todo filter out unsupported stuff? */
5846	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5847	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5848	{
5849	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5850	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5851	RT_NOREF(pSvmTransient);
5852	}
5853
5854	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5855	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5856	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5857	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5858	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5859	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5860	pDbgState->bmExitsToCheck[0],
5861	pDbgState->bmExitsToCheck[1],
5862	pDbgState->bmExitsToCheck[2],
5863	pDbgState->bmExitsToCheck[3],
5864	pDbgState->bmExitsToCheck[4]));
5865	}
5866
5867
5868	/**
5869	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5870	* appropriate.
5871	*
5872	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5873	* bitmap.
5874	*
5875	* @returns Strict VBox status code (i.e. informational status codes too).
5876	* @param pVCpu The cross context virtual CPU structure.
5877	* @param pSvmTransient The SVM-transient structure.
5878	* @param uExitCode The VM-exit code.
5879	*
5880	* @remarks The name of this function is displayed by dtrace, so keep it short
5881	* and to the point. No longer than 33 chars long, please.
5882	*/
5883	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5884	{
5885	/*
5886	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5887	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5888	*
5889	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5890	* does. Must add/change/remove both places. Same ordering, please.
5891	*
5892	* Added/removed events must also be reflected in the next section
5893	* where we dispatch dtrace events.
5894	*/
5895	bool fDtrace1 = false;
5896	bool fDtrace2 = false;
5897	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5898	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5899	uint64_t uEventArg = 0;
5900	#define SET_XCPT(a_XcptName) \
5901	do { \
5902	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5903	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5904	} while (0)
5905	#define SET_EXIT(a_EventSubName) \
5906	do { \
5907	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5908	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5909	} while (0)
5910	#define SET_BOTH(a_EventSubName) \
5911	do { \
5912	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5913	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5914	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5915	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5916	} while (0)
5917	switch (uExitCode)
5918	{
5919	case SVM_EXIT_SWINT:
5920	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5921	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5922	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5923	break;
5924
5925	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5926	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5927	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5928	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5929	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5930	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5931	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5932	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5933	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5934	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5935	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5936	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5937	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5938	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5939	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5940	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5941	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5942	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5943
5944	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5945	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5946	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5947	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5948	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5949	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5950	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5951	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5952	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5953	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5954	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5955	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5956	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5957	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5958
5959	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5960	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5961	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5962	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5963	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5964	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5965	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5966	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5967	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5968	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5969	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5970	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5971	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5972	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5973	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5974
5975	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5976	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5977	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5978	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5979	SET_BOTH(CRX_READ);
5980	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5981	break;
5982	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5983	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5984	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5985	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
5986	case SVM_EXIT_CR0_SEL_WRITE:
5987	SET_BOTH(CRX_WRITE);
5988	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
5989	break;
5990	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5991	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
5992	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
5993	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5994	SET_BOTH(DRX_READ);
5995	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
5996	break;
5997	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5998	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
5999	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6000	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6001	SET_BOTH(DRX_WRITE);
6002	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6003	break;
6004	case SVM_EXIT_MSR:
6005	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6006	SET_BOTH(WRMSR);
6007	else
6008	SET_BOTH(RDMSR);
6009	break;
6010	case SVM_EXIT_MWAIT_ARMED:
6011	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6012	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6013	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6014	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6015	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6016	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6017	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6018	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6019	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6020	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6021	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6022	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6023	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6024
6025	case SVM_EXIT_SHUTDOWN:
6026	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6027	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6028	break;
6029
6030	default:
6031	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6032	break;
6033	}
6034	#undef SET_BOTH
6035	#undef SET_EXIT
6036
6037	/*
6038	* Dtrace tracepoints go first. We do them here at once so we don't
6039	* have to copy the guest state saving and stuff a few dozen times.
6040	* Down side is that we've got to repeat the switch, though this time
6041	* we use enmEvent since the probes are a subset of what DBGF does.
6042	*/
6043	if (fDtrace1 \|\| fDtrace2)
6044	{
6045	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6046	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx; RT_NOREF(pCtx); /* Shut up Clang 13. */
6047	switch (enmEvent1)
6048	{
6049	/** @todo consider which extra parameters would be helpful for each probe. */
6050	case DBGFEVENT_END: break;
6051	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6052	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6053	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6054	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6055	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6056	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6057	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6058	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6059	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6060	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6061	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6062	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6063	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6064	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6065	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6066	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6067	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6068	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6069	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6070	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6071	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6072	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6073	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6074	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6075	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6076	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6077	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6078	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6079	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6080	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6081	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6082	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6083	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6084	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6085	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6086	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6087	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6088	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6089	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6090	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6091	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6092	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6093	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6094	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6095	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6096	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6097	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6098	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6099	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6100	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6101	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6102	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6103	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6104	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6105	}
6106	switch (enmEvent2)
6107	{
6108	/** @todo consider which extra parameters would be helpful for each probe. */
6109	case DBGFEVENT_END: break;
6110	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6111	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6112	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6113	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6114	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6115	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6116	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6117	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6118	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6119	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6120	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6121	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6122	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6123	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6124	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6125	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6126	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6127	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6128	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6129	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6130	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6131	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6132	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6133	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6134	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6135	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6136	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6137	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6138	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6139	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6140	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6141	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6142	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6144	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6145	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6146	}
6147	}
6148
6149	/*
6150	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6151	* the DBGF call will do a full check).
6152	*
6153	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6154	* Note! If we have to events, we prioritize the first, i.e. the instruction
6155	* one, in order to avoid event nesting.
6156	*/
6157	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6158	VBOXSTRICTRC rcStrict;
6159	if ( enmEvent1 != DBGFEVENT_END
6160	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6161	{
6162	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6163	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6164	}
6165	else if ( enmEvent2 != DBGFEVENT_END
6166	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6167	{
6168	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6169	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6170	}
6171	else
6172	rcStrict = VINF_SUCCESS;
6173	return rcStrict;
6174	}
6175
6176
6177	/**
6178	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6179	* debug variant.
6180	*
6181	* @returns Strict VBox status code (informational status codes included).
6182	* @param pVCpu The cross context virtual CPU structure.
6183	* @param pSvmTransient Pointer to the SVM transient structure.
6184	* @param pDbgState The runtime debug state.
6185	*/
6186	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6187	{
6188	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6189	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6190
6191	/*
6192	* Expensive (saves context) generic dtrace VM-exit probe.
6193	*/
6194	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6195	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6196	{ /* more likely */ }
6197	else
6198	{
6199	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6200	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6201	}
6202
6203	/*
6204	* Check for single stepping event if we're stepping.
6205	*/
6206	if (pVCpu->hm.s.fSingleInstruction)
6207	{
6208	switch (uExitCode)
6209	{
6210	/* Various events: */
6211	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6212	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6213	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6214	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6215	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6216	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6217	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6218	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6219	case SVM_EXIT_INTR:
6220	case SVM_EXIT_NMI:
6221	case SVM_EXIT_VINTR:
6222	case SVM_EXIT_NPF:
6223	case SVM_EXIT_AVIC_NOACCEL:
6224
6225	/* Instruction specific VM-exits: */
6226	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6227	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6228	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6229	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6230	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6231	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6232	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6233	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6234	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6235	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6236	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6237	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6238	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6239	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6240	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6241	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6242	case SVM_EXIT_CR0_SEL_WRITE:
6243	case SVM_EXIT_IDTR_READ:
6244	case SVM_EXIT_GDTR_READ:
6245	case SVM_EXIT_LDTR_READ:
6246	case SVM_EXIT_TR_READ:
6247	case SVM_EXIT_IDTR_WRITE:
6248	case SVM_EXIT_GDTR_WRITE:
6249	case SVM_EXIT_LDTR_WRITE:
6250	case SVM_EXIT_TR_WRITE:
6251	case SVM_EXIT_RDTSC:
6252	case SVM_EXIT_RDPMC:
6253	case SVM_EXIT_PUSHF:
6254	case SVM_EXIT_POPF:
6255	case SVM_EXIT_CPUID:
6256	case SVM_EXIT_RSM:
6257	case SVM_EXIT_IRET:
6258	case SVM_EXIT_SWINT:
6259	case SVM_EXIT_INVD:
6260	case SVM_EXIT_PAUSE:
6261	case SVM_EXIT_HLT:
6262	case SVM_EXIT_INVLPG:
6263	case SVM_EXIT_INVLPGA:
6264	case SVM_EXIT_IOIO:
6265	case SVM_EXIT_MSR:
6266	case SVM_EXIT_TASK_SWITCH:
6267	case SVM_EXIT_VMRUN:
6268	case SVM_EXIT_VMMCALL:
6269	case SVM_EXIT_VMLOAD:
6270	case SVM_EXIT_VMSAVE:
6271	case SVM_EXIT_STGI:
6272	case SVM_EXIT_CLGI:
6273	case SVM_EXIT_SKINIT:
6274	case SVM_EXIT_RDTSCP:
6275	case SVM_EXIT_ICEBP:
6276	case SVM_EXIT_WBINVD:
6277	case SVM_EXIT_MONITOR:
6278	case SVM_EXIT_MWAIT:
6279	case SVM_EXIT_MWAIT_ARMED:
6280	case SVM_EXIT_XSETBV:
6281	case SVM_EXIT_RDPRU:
6282	case SVM_EXIT_WRITE_EFER_TRAP:
6283	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6284	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6285	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6286	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6287	case SVM_EXIT_MCOMMIT:
6288	{
6289	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6290	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6291	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6292	{
6293	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6294	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6295	return VINF_EM_DBG_STEPPED;
6296	}
6297	break;
6298	}
6299
6300	/* Errors and unexpected events: */
6301	case SVM_EXIT_FERR_FREEZE:
6302	case SVM_EXIT_SHUTDOWN:
6303	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6304	break;
6305
6306	case SVM_EXIT_SMI:
6307	case SVM_EXIT_INIT:
6308	default:
6309	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6310	break;
6311	}
6312	}
6313
6314	/*
6315	* Check for debugger event breakpoints and dtrace probes.
6316	*/
6317	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6318	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6319	{
6320	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6321	if (rcStrict != VINF_SUCCESS)
6322	{
6323	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6324	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6325	return rcStrict;
6326	}
6327	}
6328
6329	/*
6330	* Normal processing.
6331	*/
6332	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6333	}
6334
6335
6336	/**
6337	* Runs the guest code using AMD-V in single step mode.
6338	*
6339	* @returns Strict VBox status code.
6340	* @param pVCpu The cross context virtual CPU structure.
6341	* @param pcLoops Pointer to the number of executed loops.
6342	*/
6343	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6344	{
6345	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6346	Assert(pcLoops);
6347	Assert(*pcLoops <= cMaxResumeLoops);
6348
6349	SVMTRANSIENT SvmTransient;
6350	RT_ZERO(SvmTransient);
6351	SvmTransient.fUpdateTscOffsetting = true;
6352	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6353
6354	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6355
6356	/* Set HMCPU indicators. */
6357	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6358	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6359	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6360	pVCpu->hmr0.s.fUsingDebugLoop = true;
6361
6362	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6363	SVMRUNDBGSTATE DbgState;
6364	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6365	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6366
6367	/*
6368	* The loop.
6369	*/
6370	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6371	for (;;)
6372	{
6373	Assert(!HMR0SuspendPending());
6374	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6375	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6376	(unsigned)RTMpCpuId(), *pcLoops));
6377	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6378
6379	/* Set up VM-execution controls the next two can respond to. */
6380	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6381
6382	/* Preparatory work for running nested-guest code, this may force us to return to
6383	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6384	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6385	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6386	if (rc != VINF_SUCCESS)
6387	break;
6388
6389	/*
6390	* No longjmps to ring-3 from this point on!!!
6391	*
6392	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6393	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6394	*/
6395	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6396
6397	/* Override any obnoxious code in the above two calls. */
6398	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6399	#if 0
6400	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6401	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6402	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6403	#endif
6404
6405	/*
6406	* Finally execute guest code.
6407	*/
6408	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6409
6410	/* Restore any residual host-state and save any bits shared between host and guest
6411	into the guest-CPU state. Re-enables interrupts! */
6412	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6413	#if 0
6414	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6415	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6416	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6417	#endif
6418
6419	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6420	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6421	{ /* very likely*/ }
6422	else
6423	{
6424	if (rc == VINF_SUCCESS)
6425	rc = VERR_SVM_INVALID_GUEST_STATE;
6426	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6427	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6428	return rc;
6429	}
6430
6431	/* Handle the #VMEXIT. */
6432	HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6433	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6434	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6435	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6436	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6437	if (rc != VINF_SUCCESS)
6438	break;
6439	if (++(*pcLoops) >= cMaxResumeLoops)
6440	{
6441	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6442	rc = VINF_EM_RAW_INTERRUPT;
6443	break;
6444	}
6445
6446	/*
6447	* Stepping: Did the RIP change, if so, consider it a single step.
6448	* Otherwise, make sure one of the TFs gets set.
6449	*/
6450	if (fStepping)
6451	{
6452	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6453	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6454	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6455	{
6456	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6457	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6458	rc = VINF_EM_DBG_STEPPED;
6459	break;
6460	}
6461	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6462	}
6463
6464	/*
6465	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6466	* Revert the state changes afterware so we can drop intercepts no longer needed.
6467	*/
6468	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6469	{
6470	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6471	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6472	}
6473	}
6474
6475	/*
6476	* Clear the X86_EFL_TF if necessary.
6477	*/
6478	if (pVCpu->hmr0.s.fClearTrapFlag)
6479	{
6480	pVCpu->hmr0.s.fClearTrapFlag = false;
6481	pCtx->eflags.Bits.u1TF = 0;
6482	}
6483
6484	/* Restore HMCPU indicators. */
6485	pVCpu->hmr0.s.fUsingDebugLoop = false;
6486	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6487	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6488
6489	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6490	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6491
6492	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6493	return rc;
6494	}
6495
6496	/** @} */
6497
6498	#undef VMEXIT_CALL_RET
6499
6500
6501	#ifdef VBOX_STRICT
6502	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6503	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6504	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6505
6506	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6507	do \
6508	{ \
6509	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6510	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6511	} while (0)
6512
6513	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6514	do { \
6515	AssertPtr((a_pVCpu)); \
6516	AssertPtr((a_pSvmTransient)); \
6517	Assert(ASMIntAreEnabled()); \
6518	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6519	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6520	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6521	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6522	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6523	HMSVM_ASSERT_PREEMPT_CPUID(); \
6524	} while (0)
6525	#else
6526	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6527	do { \
6528	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6529	} while (0)
6530	#endif
6531
6532
6533	/**
6534	* Gets the IEM exception flags for the specified SVM event.
6535	*
6536	* @returns The IEM exception flags.
6537	* @param pEvent Pointer to the SVM event.
6538	*
6539	* @remarks This function currently only constructs flags required for
6540	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6541	* and CR2 aspects of an exception are not included).
6542	*/
6543	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6544	{
6545	uint8_t const uEventType = pEvent->n.u3Type;
6546	uint32_t fIemXcptFlags;
6547	switch (uEventType)
6548	{
6549	case SVM_EVENT_EXCEPTION:
6550	/*
6551	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6552	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6553	*/
6554	if (pEvent->n.u8Vector == X86_XCPT_BP)
6555	{
6556	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6557	break;
6558	}
6559	if (pEvent->n.u8Vector == X86_XCPT_OF)
6560	{
6561	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6562	break;
6563	}
6564	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6565	RT_FALL_THRU();
6566	case SVM_EVENT_NMI:
6567	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6568	break;
6569
6570	case SVM_EVENT_EXTERNAL_IRQ:
6571	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6572	break;
6573
6574	case SVM_EVENT_SOFTWARE_INT:
6575	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6576	break;
6577
6578	default:
6579	fIemXcptFlags = 0;
6580	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6581	break;
6582	}
6583	return fIemXcptFlags;
6584	}
6585
6586
6587	/**
6588	* Handle a condition that occurred while delivering an event through the guest
6589	* IDT.
6590	*
6591	* @returns VBox status code (informational error codes included).
6592	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6593	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6594	* continue execution of the guest which will delivery the \#DF.
6595	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6596	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6597	*
6598	* @param pVCpu The cross context virtual CPU structure.
6599	* @param pSvmTransient Pointer to the SVM transient structure.
6600	*
6601	* @remarks No-long-jump zone!!!
6602	*/
6603	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6604	{
6605	/** @todo r=bird: Looks like this is called on many exits and we start by
6606	* loading CR2 on the offchance that we actually have work to do here.
6607	*
6608	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6609	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6610	*
6611	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6612	*/
6613	int rc = VINF_SUCCESS;
6614	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6615	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6616
6617	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6618	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6619	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6620
6621	/*
6622	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6623	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6624	*
6625	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6626	*/
6627	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6628	{
6629	IEMXCPTRAISE enmRaise;
6630	IEMXCPTRAISEINFO fRaiseInfo;
6631	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6632	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6633	if (fExitIsHwXcpt)
6634	{
6635	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6636	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6637	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6638	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6639	}
6640	else
6641	{
6642	/*
6643	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6644	* then we end up here.
6645	*
6646	* If the event was:
6647	* - a software interrupt, we can re-execute the instruction which will
6648	* regenerate the event.
6649	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6650	* - a hardware exception or external interrupt, we re-inject it.
6651	*/
6652	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6653	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6654	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6655	else
6656	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6657	}
6658
6659	switch (enmRaise)
6660	{
6661	case IEMXCPTRAISE_CURRENT_XCPT:
6662	case IEMXCPTRAISE_PREV_EVENT:
6663	{
6664	/* For software interrupts, we shall re-execute the instruction. */
6665	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6666	{
6667	RTGCUINTPTR GCPtrFaultAddress = 0;
6668
6669	/* If we are re-injecting an NMI, clear NMI blocking. */
6670	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6671	CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
6672
6673	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6674	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6675	{
6676	pSvmTransient->fVectoringPF = true;
6677	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6678	pVCpu->cpum.GstCtx.cr2));
6679	}
6680	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6681	&& uIdtVector == X86_XCPT_PF)
6682	{
6683	/*
6684	* If the previous exception was a #PF, we need to recover the CR2 value.
6685	* This can't happen with shadow paging.
6686	*/
6687	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6688	}
6689
6690	/*
6691	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6692	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6693	*/
6694	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6695	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6696	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6697
6698	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6699	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6700	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6701	}
6702	break;
6703	}
6704
6705	case IEMXCPTRAISE_REEXEC_INSTR:
6706	{
6707	Assert(rc == VINF_SUCCESS);
6708	break;
6709	}
6710
6711	case IEMXCPTRAISE_DOUBLE_FAULT:
6712	{
6713	/*
6714	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6715	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6716	* converted into a #DF.
6717	*/
6718	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6719	{
6720	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6721	pSvmTransient->fVectoringDoublePF = true;
6722	Assert(rc == VINF_SUCCESS);
6723	}
6724	else
6725	{
6726	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6727	hmR0SvmSetPendingXcptDF(pVCpu);
6728	rc = VINF_HM_DOUBLE_FAULT;
6729	}
6730	break;
6731	}
6732
6733	case IEMXCPTRAISE_TRIPLE_FAULT:
6734	{
6735	rc = VINF_EM_RESET;
6736	break;
6737	}
6738
6739	case IEMXCPTRAISE_CPU_HANG:
6740	{
6741	rc = VERR_EM_GUEST_CPU_HANG;
6742	break;
6743	}
6744
6745	default:
6746	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6747	}
6748	}
6749	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6750	return rc;
6751	}
6752
6753
6754	/**
6755	* Advances the guest RIP by the number of bytes specified in @a cb.
6756	*
6757	* @param pVCpu The cross context virtual CPU structure.
6758	* @param cb RIP increment value in bytes.
6759	*/
6760	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6761	{
6762	pVCpu->cpum.GstCtx.rip += cb;
6763	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
6764	/** @todo clear RF. */
6765	}
6766
6767
6768	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6769	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6770	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6771
6772	/** @name \#VMEXIT handlers.
6773	* @{
6774	*/
6775
6776	/**
6777	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6778	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6779	*/
6780	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6781	{
6782	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6783
6784	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6785	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6786	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6787	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6788
6789	/*
6790	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6791	* signal -before- the timer fires if the current interrupt is our own timer or a some
6792	* other host interrupt. We also cannot examine what interrupt it is until the host
6793	* actually take the interrupt.
6794	*
6795	* Going back to executing guest code here unconditionally causes random scheduling
6796	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6797	*/
6798	return VINF_EM_RAW_INTERRUPT;
6799	}
6800
6801
6802	/**
6803	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6804	*/
6805	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6806	{
6807	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6808
6809	VBOXSTRICTRC rcStrict;
6810	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6811	if (fSupportsNextRipSave)
6812	{
6813	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6814	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6815	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6816	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6817	}
6818	else
6819	{
6820	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6821	rcStrict = IEMExecOne(pVCpu);
6822	}
6823
6824	if (rcStrict == VINF_IEM_RAISED_XCPT)
6825	{
6826	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6827	rcStrict = VINF_SUCCESS;
6828	}
6829	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6830	return rcStrict;
6831	}
6832
6833
6834	/**
6835	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6836	*/
6837	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6838	{
6839	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6840
6841	VBOXSTRICTRC rcStrict;
6842	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6843	if (fSupportsNextRipSave)
6844	{
6845	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6846	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6847	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6848	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6849	}
6850	else
6851	{
6852	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6853	rcStrict = IEMExecOne(pVCpu);
6854	}
6855
6856	if (rcStrict == VINF_IEM_RAISED_XCPT)
6857	{
6858	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6859	rcStrict = VINF_SUCCESS;
6860	}
6861	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6862	return rcStrict;
6863	}
6864
6865
6866	/**
6867	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6868	*/
6869	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6870	{
6871	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6872
6873	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6874	VBOXSTRICTRC rcStrict;
6875	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6876	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6877	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6878	if (!pExitRec)
6879	{
6880	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6881	if (fSupportsNextRipSave)
6882	{
6883	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6884	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6885	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6886	}
6887	else
6888	{
6889	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6890	rcStrict = IEMExecOne(pVCpu);
6891	}
6892
6893	if (rcStrict == VINF_IEM_RAISED_XCPT)
6894	{
6895	CPUM_ASSERT_NOT_EXTRN(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
6896	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6897	rcStrict = VINF_SUCCESS;
6898	}
6899	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6900	}
6901	else
6902	{
6903	/*
6904	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6905	*/
6906	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6907
6908	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6909	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6910
6911	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6912
6913	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6914	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6915	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6916	}
6917	return rcStrict;
6918	}
6919
6920
6921	/**
6922	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6923	*/
6924	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6925	{
6926	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6927
6928	VBOXSTRICTRC rcStrict;
6929	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6930	if (fSupportsNextRipSave)
6931	{
6932	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6933	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6934	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6935	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6936	}
6937	else
6938	{
6939	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6940	rcStrict = IEMExecOne(pVCpu);
6941	}
6942
6943	if (rcStrict == VINF_SUCCESS)
6944	pSvmTransient->fUpdateTscOffsetting = true;
6945	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6946	{
6947	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6948	rcStrict = VINF_SUCCESS;
6949	}
6950	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6951	return rcStrict;
6952	}
6953
6954
6955	/**
6956	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6957	*/
6958	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6959	{
6960	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6961
6962	VBOXSTRICTRC rcStrict;
6963	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6964	if (fSupportsNextRipSave)
6965	{
6966	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6967	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6968	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6969	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6970	}
6971	else
6972	{
6973	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6974	rcStrict = IEMExecOne(pVCpu);
6975	}
6976
6977	if (rcStrict == VINF_SUCCESS)
6978	pSvmTransient->fUpdateTscOffsetting = true;
6979	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6980	{
6981	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6982	rcStrict = VINF_SUCCESS;
6983	}
6984	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6985	return rcStrict;
6986	}
6987
6988
6989	/**
6990	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6991	*/
6992	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6993	{
6994	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6995
6996	VBOXSTRICTRC rcStrict;
6997	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6998	if (fSupportsNextRipSave)
6999	{
7000	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
7001	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7002	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7003	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7004	}
7005	else
7006	{
7007	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7008	rcStrict = IEMExecOne(pVCpu);
7009	}
7010
7011	if (rcStrict == VINF_IEM_RAISED_XCPT)
7012	{
7013	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7014	rcStrict = VINF_SUCCESS;
7015	}
7016	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7017	return rcStrict;
7018	}
7019
7020
7021	/**
7022	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7023	*/
7024	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7025	{
7026	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7027	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7028
7029	VBOXSTRICTRC rcStrict;
7030	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7031	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7032	if ( fSupportsDecodeAssists
7033	&& fSupportsNextRipSave)
7034	{
7035	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7036	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7037	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7038	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7039	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7040	}
7041	else
7042	{
7043	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7044	rcStrict = IEMExecOne(pVCpu);
7045	}
7046
7047	if (rcStrict == VINF_IEM_RAISED_XCPT)
7048	{
7049	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7050	rcStrict = VINF_SUCCESS;
7051	}
7052	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7053	return VBOXSTRICTRC_VAL(rcStrict);
7054	}
7055
7056
7057	/**
7058	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7059	*/
7060	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7061	{
7062	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7063
7064	VBOXSTRICTRC rcStrict;
7065	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7066	if (fSupportsNextRipSave)
7067	{
7068	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7069	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7070	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7071	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7072	}
7073	else
7074	{
7075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7076	rcStrict = IEMExecOne(pVCpu);
7077	}
7078
7079	if ( rcStrict == VINF_EM_HALT
7080	\|\| rcStrict == VINF_SUCCESS)
7081	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7082	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7083	{
7084	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7085	rcStrict = VINF_SUCCESS;
7086	}
7087	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7088	if (rcStrict != VINF_SUCCESS)
7089	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7090	return VBOXSTRICTRC_VAL(rcStrict);;
7091	}
7092
7093
7094	/**
7095	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7096	*/
7097	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7098	{
7099	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7100
7101	/*
7102	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7103	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7104	* segment override prefix or other prefixes might be used, in which case we fallback to
7105	* IEMExecOne() to figure out.
7106	*/
7107	VBOXSTRICTRC rcStrict;
7108	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7109	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7110	if (cbInstr)
7111	{
7112	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7113	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7114	}
7115	else
7116	{
7117	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7118	rcStrict = IEMExecOne(pVCpu);
7119	}
7120
7121	if (rcStrict == VINF_IEM_RAISED_XCPT)
7122	{
7123	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7124	rcStrict = VINF_SUCCESS;
7125	}
7126	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7127	return rcStrict;
7128	}
7129
7130
7131	/**
7132	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7133	*/
7134	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7135	{
7136	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7137
7138	VBOXSTRICTRC rcStrict;
7139	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7140	if (fSupportsNextRipSave)
7141	{
7142	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7143	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7144	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7145	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7146	}
7147	else
7148	{
7149	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7150	rcStrict = IEMExecOne(pVCpu);
7151	}
7152
7153	if ( rcStrict == VINF_EM_HALT
7154	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7155	rcStrict = VINF_SUCCESS;
7156	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7157	{
7158	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7159	rcStrict = VINF_SUCCESS;
7160	}
7161	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7162	return rcStrict;
7163	}
7164
7165
7166	/**
7167	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7168	* \#VMEXIT.
7169	*/
7170	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7171	{
7172	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7173	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7174	return VINF_EM_RESET;
7175	}
7176
7177
7178	/**
7179	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7180	*/
7181	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7182	{
7183	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7184	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7185	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7186	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7187	RT_NOREF(pVmcb);
7188	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7189	return VERR_SVM_UNEXPECTED_EXIT;
7190	}
7191
7192
7193	/**
7194	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7195	*/
7196	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7197	{
7198	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7199
7200	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7201	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7202	#ifdef VBOX_WITH_STATISTICS
7203	switch (pSvmTransient->u64ExitCode)
7204	{
7205	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7206	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7207	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7208	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7209	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7210	}
7211	#endif
7212
7213	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7214	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7215	if ( fSupportsDecodeAssists
7216	&& fSupportsNextRipSave)
7217	{
7218	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7219	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7220	if (fMovCRx)
7221	{
7222	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7223	\| CPUMCTX_EXTRN_APIC_TPR);
7224	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7225	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7226	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7227	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7228	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7229	return VBOXSTRICTRC_VAL(rcStrict);
7230	}
7231	/* else: SMSW instruction, fall back below to IEM for this. */
7232	}
7233
7234	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7235	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7236	AssertMsg( rcStrict == VINF_SUCCESS
7237	\|\| rcStrict == VINF_PGM_SYNC_CR3
7238	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7239	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7240	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7241	if (rcStrict == VINF_IEM_RAISED_XCPT)
7242	{
7243	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7244	rcStrict = VINF_SUCCESS;
7245	}
7246	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7247	return rcStrict;
7248	}
7249
7250
7251	/**
7252	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7253	*/
7254	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7255	{
7256	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7257
7258	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7259	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7260	Assert(iCrReg <= 15);
7261
7262	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7263	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7264	bool fDecodedInstr = false;
7265	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7266	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7267	if ( fSupportsDecodeAssists
7268	&& fSupportsNextRipSave)
7269	{
7270	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7271	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7272	if (fMovCRx)
7273	{
7274	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7275	\| CPUMCTX_EXTRN_APIC_TPR);
7276	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7277	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7278	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7279	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7280	fDecodedInstr = true;
7281	}
7282	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7283	}
7284
7285	if (!fDecodedInstr)
7286	{
7287	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7288	Log4Func(("iCrReg=%#x\n", iCrReg));
7289	rcStrict = IEMExecOne(pVCpu);
7290	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7291	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7292	rcStrict = VERR_EM_INTERPRETER;
7293	}
7294
7295	if (rcStrict == VINF_SUCCESS)
7296	{
7297	switch (iCrReg)
7298	{
7299	case 0:
7300	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7302	break;
7303
7304	case 2:
7305	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7307	break;
7308
7309	case 3:
7310	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7312	break;
7313
7314	case 4:
7315	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7317	break;
7318
7319	case 8:
7320	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7321	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7322	break;
7323
7324	default:
7325	{
7326	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7327	pSvmTransient->u64ExitCode, iCrReg));
7328	break;
7329	}
7330	}
7331	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7332	}
7333	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7334	{
7335	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7336	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7337	rcStrict = VINF_SUCCESS;
7338	}
7339	else
7340	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7341	return rcStrict;
7342	}
7343
7344
7345	/**
7346	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7347	*
7348	* @returns Strict VBox status code.
7349	* @param pVCpu The cross context virtual CPU structure.
7350	* @param pVmcb Pointer to the VM control block.
7351	*/
7352	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7353	{
7354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7355	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7356
7357	VBOXSTRICTRC rcStrict;
7358	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7359	if (fSupportsNextRipSave)
7360	{
7361	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7362	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7363	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7364	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7365	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7366	}
7367	else
7368	{
7369	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7370	rcStrict = IEMExecOne(pVCpu);
7371	}
7372
7373	AssertMsg( rcStrict == VINF_SUCCESS
7374	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7375	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7376	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7377
7378	if (rcStrict == VINF_IEM_RAISED_XCPT)
7379	{
7380	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7381	rcStrict = VINF_SUCCESS;
7382	}
7383	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7384	return rcStrict;
7385	}
7386
7387
7388	/**
7389	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7390	*
7391	* @returns Strict VBox status code.
7392	* @param pVCpu The cross context virtual CPU structure.
7393	* @param pVmcb Pointer to the VM control block.
7394	* @param pSvmTransient Pointer to the SVM-transient structure.
7395	*/
7396	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7397	{
7398	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7399	uint32_t const idMsr = pCtx->ecx;
7400	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7401	Log4Func(("idMsr=%#RX32\n", idMsr));
7402
7403	/*
7404	* Handle TPR patching MSR writes.
7405	* We utilitize the LSTAR MSR for patching.
7406	*/
7407	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7408	if ( idMsr == MSR_K8_LSTAR
7409	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7410	{
7411	unsigned cbInstr;
7412	if (fSupportsNextRipSave)
7413	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7414	else
7415	{
7416	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
7417	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
7418	if ( rc == VINF_SUCCESS
7419	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7420	Assert(cbInstr > 0);
7421	else
7422	cbInstr = 0;
7423	}
7424
7425	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7426	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7427	{
7428	int rc = PDMApicSetTpr(pVCpu, pCtx->eax & 0xff);
7429	AssertRCReturn(rc, rc);
7430	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7431	}
7432
7433	int rc = VINF_SUCCESS;
7434	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7435	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7436	return rc;
7437	}
7438
7439	/*
7440	* Handle regular MSR writes.
7441	*/
7442	VBOXSTRICTRC rcStrict;
7443	if (fSupportsNextRipSave)
7444	{
7445	/** @todo Optimize this: We don't need to get much of the MSR state here
7446	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7447	* clear the applicable extern flags. */
7448	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7449	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7450	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7451	}
7452	else
7453	{
7454	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7455	rcStrict = IEMExecOne(pVCpu);
7456	}
7457
7458	AssertMsg( rcStrict == VINF_SUCCESS
7459	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7460	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7461	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7462
7463	if (rcStrict == VINF_SUCCESS)
7464	{
7465	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7466	if ( idMsr >= MSR_IA32_X2APIC_START
7467	&& idMsr <= MSR_IA32_X2APIC_END)
7468	{
7469	/*
7470	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7471	* When full APIC register virtualization is implemented we'll have to make sure
7472	* APIC state is saved from the VMCB before IEM changes it.
7473	*/
7474	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7475	}
7476	else
7477	{
7478	switch (idMsr)
7479	{
7480	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7481	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7482	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7483	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7484	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7485	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7486	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7487	}
7488	}
7489	}
7490	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7491	{
7492	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7493	rcStrict = VINF_SUCCESS;
7494	}
7495	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7496	return rcStrict;
7497	}
7498
7499
7500	/**
7501	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7502	* \#VMEXIT.
7503	*/
7504	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7505	{
7506	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7507
7508	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7509	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7510	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7511
7512	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7513	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7514	}
7515
7516
7517	/**
7518	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7519	*/
7520	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7521	{
7522	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7523	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7524
7525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7526
7527	/** @todo Stepping with nested-guest. */
7528	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7529	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7530	{
7531	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7532	if (pSvmTransient->fWasGuestDebugStateActive)
7533	{
7534	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7535	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7536	return VERR_SVM_UNEXPECTED_EXIT;
7537	}
7538
7539	/*
7540	* Lazy DR0-3 loading.
7541	*/
7542	if (!pSvmTransient->fWasHyperDebugStateActive)
7543	{
7544	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7545	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7546
7547	/* Don't intercept DRx read and writes. */
7548	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7549	pVmcb->ctrl.u16InterceptRdDRx = 0;
7550	pVmcb->ctrl.u16InterceptWrDRx = 0;
7551	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7552
7553	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7554	VMMRZCallRing3Disable(pVCpu);
7555	HM_DISABLE_PREEMPT(pVCpu);
7556
7557	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7558	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7559	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7560
7561	HM_RESTORE_PREEMPT();
7562	VMMRZCallRing3Enable(pVCpu);
7563
7564	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7565	return VINF_SUCCESS;
7566	}
7567	}
7568
7569	/*
7570	* Interpret the read/writing of DRx.
7571	*/
7572	/** @todo Decode assist. */
7573	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
7574	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7575	if (RT_LIKELY(rc == VINF_SUCCESS))
7576	{
7577	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7578	/** @todo CPUM should set this flag! */
7579	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7580	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7581	}
7582	else
7583	Assert(rc == VERR_EM_INTERPRETER);
7584	return rc;
7585	}
7586
7587
7588	/**
7589	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7590	*/
7591	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7592	{
7593	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7594	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7595	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7597	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7598	return rc;
7599	}
7600
7601
7602	/**
7603	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7604	*/
7605	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7606	{
7607	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7608	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7609
7610	/** @todo decode assists... */
7611	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7612	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7613	{
7614	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7615	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7616	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7617	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7618	{
7619	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7620	hmR0SvmUpdateVmRunFunction(pVCpu);
7621	}
7622	}
7623	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7624	{
7625	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7626	rcStrict = VINF_SUCCESS;
7627	}
7628	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7629	return rcStrict;
7630	}
7631
7632
7633	/**
7634	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7635	*/
7636	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7637	{
7638	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7639	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7640
7641	/* I/O operation lookup arrays. */
7642	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7643	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7644	the result (in AL/AX/EAX). */
7645	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7646	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7647	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7648
7649	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7650
7651	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7652	SVMIOIOEXITINFO IoExitInfo;
7653	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7654	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7655	uint32_t cbValue = s_aIOSize[uIOWidth];
7656	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7657
7658	if (RT_UNLIKELY(!cbValue))
7659	{
7660	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7661	return VERR_EM_INTERPRETER;
7662	}
7663
7664	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7665	VBOXSTRICTRC rcStrict;
7666	PCEMEXITREC pExitRec = NULL;
7667	if ( !pVCpu->hm.s.fSingleInstruction
7668	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7669	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7670	!IoExitInfo.n.u1Str
7671	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7672	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7673	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7674	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7675	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7676	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7677	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7678	if (!pExitRec)
7679	{
7680	bool fUpdateRipAlready = false;
7681	if (IoExitInfo.n.u1Str)
7682	{
7683	/* INS/OUTS - I/O String instruction. */
7684	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7685	* in EXITINFO1? Investigate once this thing is up and running. */
7686	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7687	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7688	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7689	static IEMMODE const s_aenmAddrMode[8] =
7690	{
7691	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7692	};
7693	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7694	if (enmAddrMode != (IEMMODE)-1)
7695	{
7696	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7697	if (cbInstr <= 15 && cbInstr >= 1)
7698	{
7699	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7700	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7701	{
7702	/* Don't know exactly how to detect whether u3Seg is valid, currently
7703	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7704	2384 Opterons when only checking NRIP. */
7705	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7706	if ( fSupportsNextRipSave
7707	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7708	{
7709	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7710	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7711	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7712	IoExitInfo.n.u3Seg, true /fIoChecked/);
7713	}
7714	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7715	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7716	X86_SREG_DS, true /fIoChecked/);
7717	else
7718	rcStrict = IEMExecOne(pVCpu);
7719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7720	}
7721	else
7722	{
7723	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7724	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7725	true /fIoChecked/);
7726	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7727	}
7728	}
7729	else
7730	{
7731	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7732	rcStrict = IEMExecOne(pVCpu);
7733	}
7734	}
7735	else
7736	{
7737	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7738	rcStrict = IEMExecOne(pVCpu);
7739	}
7740	fUpdateRipAlready = true;
7741	if (rcStrict == VINF_IEM_RAISED_XCPT)
7742	{
7743	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7744	rcStrict = VINF_SUCCESS;
7745	}
7746	}
7747	else
7748	{
7749	/* IN/OUT - I/O instruction. */
7750	Assert(!IoExitInfo.n.u1Rep);
7751
7752	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7753	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7754	{
7755	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7756	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7757	&& !pCtx->eflags.Bits.u1TF)
7758	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7760	}
7761	else
7762	{
7763	rcStrict = VERR_GCM_NOT_HANDLED;
7764	if (GCMIsInterceptingIOPortRead(pVCpu, IoExitInfo.n.u16Port, cbValue))
7765	{
7766	rcStrict = GCMInterceptedIOPortRead(pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
7767	if (rcStrict == VINF_GCM_HANDLED_ADVANCE_RIP \|\| rcStrict == VINF_GCM_HANDLED)
7768	{
7769	fUpdateRipAlready = rcStrict != VINF_GCM_HANDLED_ADVANCE_RIP;
7770	rcStrict = VINF_SUCCESS;
7771	}
7772	else
7773	Assert(rcStrict == VERR_GCM_NOT_HANDLED);
7774	}
7775
7776	if (RT_LIKELY(rcStrict == VERR_GCM_NOT_HANDLED))
7777	{
7778	uint32_t u32Val = 0;
7779	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7780	if (IOM_SUCCESS(rcStrict))
7781	{
7782	/* Save result of I/O IN instr. in AL/AX/EAX. */
7783	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7784	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7785	}
7786	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7787	&& !pCtx->eflags.Bits.u1TF)
7788	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7789	}
7790
7791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7792	}
7793	}
7794
7795	if (IOM_SUCCESS(rcStrict))
7796	{
7797	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7798	if (!fUpdateRipAlready)
7799	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7800
7801	/*
7802	* If any I/O breakpoints are armed, we need to check if one triggered
7803	* and take appropriate action.
7804	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7805	*/
7806	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7807	* execution engines about whether hyper BPs and such are pending. */
7808	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7809	uint32_t const uDr7 = pCtx->dr[7];
7810	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7811	&& X86_DR7_ANY_RW_IO(uDr7)
7812	&& (pCtx->cr4 & X86_CR4_DE))
7813	\|\| DBGFBpIsHwIoArmed(pVM)))
7814	{
7815	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7816	VMMRZCallRing3Disable(pVCpu);
7817	HM_DISABLE_PREEMPT(pVCpu);
7818
7819	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7820	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7821
7822	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7823	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7824	{
7825	/* Raise #DB. */
7826	pVmcb->guest.u64DR6 = pCtx->dr[6];
7827	pVmcb->guest.u64DR7 = pCtx->dr[7];
7828	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7829	hmR0SvmSetPendingXcptDB(pVCpu);
7830	}
7831	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7832	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7833	else if ( rcStrict2 != VINF_SUCCESS
7834	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7835	rcStrict = rcStrict2;
7836	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7837
7838	HM_RESTORE_PREEMPT();
7839	VMMRZCallRing3Enable(pVCpu);
7840	}
7841
7842	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7843	}
7844	#ifdef VBOX_STRICT
7845	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7846	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7847	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7848	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7849	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7850	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7851	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7852	else
7853	{
7854	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7855	* statuses, that the VMM device and some others may return. See
7856	* IOM_SUCCESS() for guidance. */
7857	AssertMsg( RT_FAILURE(rcStrict)
7858	\|\| rcStrict == VINF_SUCCESS
7859	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7860	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7861	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7862	\|\| rcStrict == VINF_EM_DBG_STEPPED
7863	\|\| rcStrict == VINF_EM_RAW_TO_R3
7864	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7865	}
7866	#endif
7867	}
7868	else
7869	{
7870	/*
7871	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7872	*/
7873	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7874	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7875	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7876	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7877	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7878	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7879	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7880
7881	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7882	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7883
7884	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7885	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7886	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7887	}
7888	return rcStrict;
7889	}
7890
7891
7892	/**
7893	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7894	*/
7895	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7896	{
7897	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7898	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7899	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7900
7901	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7902	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7903	Assert(pVM->hmr0.s.fNestedPaging);
7904
7905	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7906	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7907	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7908	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7909	thus intentionally not copied into u32ErrCode. */
7910
7911	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7912	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7913
7914	/*
7915	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7916	*/
7917	if ( pVM->hm.s.fTprPatchingAllowed
7918	&& (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7919	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7920	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7921	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7922	&& !CPUMIsGuestInLongModeEx(pCtx)
7923	&& !CPUMGetGuestCPL(pVCpu)
7924	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7925	{
7926	RTGCPHYS GCPhysApicBase = PDMApicGetBaseMsrNoCheck(pVCpu);
7927	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
7928
7929	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7930	{
7931	/* Only attempt to patch the instruction once. */
7932	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7933	if (!pPatch)
7934	return VINF_EM_HM_PATCH_TPR_INSTR;
7935	}
7936	}
7937
7938	/*
7939	* Determine the nested paging mode.
7940	*/
7941	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7942	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7943
7944	/*
7945	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7946	*/
7947	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7948	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7949	{
7950	/*
7951	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7952	* injecting the original pending event would most likely cause the same MMIO #NPF.
7953	*/
7954	if (pVCpu->hm.s.Event.fPending)
7955	{
7956	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7957	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7958	}
7959
7960	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7961	VBOXSTRICTRC rcStrict;
7962	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7963	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7964	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7965	if (!pExitRec)
7966	{
7967
7968	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, pCtx, GCPhysFaultAddr, u32ErrCode);
7969
7970	/*
7971	* If we succeed, resume guest execution.
7972	*
7973	* If we fail in interpreting the instruction because we couldn't get the guest
7974	* physical address of the page containing the instruction via the guest's page
7975	* tables (we would invalidate the guest page in the host TLB), resume execution
7976	* which would cause a guest page fault to let the guest handle this weird case.
7977	*
7978	* See @bugref{6043}.
7979	*/
7980	if ( rcStrict == VINF_SUCCESS
7981	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7982	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7983	{
7984	/* Successfully handled MMIO operation. */
7985	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7986	rcStrict = VINF_SUCCESS;
7987	}
7988	}
7989	else
7990	{
7991	/*
7992	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7993	*/
7994	Assert(pCtx == &pVCpu->cpum.GstCtx);
7995	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7996	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7997	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7998
7999	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8000	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8001
8002	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8003	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8004	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8005	}
8006	return rcStrict;
8007	}
8008
8009	/*
8010	* Nested page-fault.
8011	*/
8012	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
8013	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, pCtx, GCPhysFaultAddr);
8014	TRPMResetTrap(pVCpu);
8015
8016	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8017
8018	/*
8019	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8020	*/
8021	if ( rc == VINF_SUCCESS
8022	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8023	\|\| rc == VERR_PAGE_NOT_PRESENT)
8024	{
8025	/* We've successfully synced our shadow page tables. */
8026	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8027	rc = VINF_SUCCESS;
8028	}
8029
8030	/*
8031	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8032	* re-inject the original event.
8033	*/
8034	if (pVCpu->hm.s.Event.fPending)
8035	{
8036	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8037
8038	/*
8039	* If the #NPF handler requested emulation of the instruction, ignore it.
8040	* We need to re-inject the original event so as to not lose it.
8041	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8042	* LiveCD is broken for other reasons).
8043	*/
8044	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8045	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8046	}
8047
8048	return rc;
8049	}
8050
8051
8052	/**
8053	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8054	* \#VMEXIT.
8055	*/
8056	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8057	{
8058	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8059	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8060
8061	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8062	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8063	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8064
8065	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8066	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8067	return VINF_SUCCESS;
8068	}
8069
8070
8071	/**
8072	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8073	* \#VMEXIT.
8074	*/
8075	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8076	{
8077	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8078	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8079
8080	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8081	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8082	#endif
8083
8084	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8085	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8086	{
8087	/*
8088	* AMD-V provides us with the exception which caused the TS; we collect
8089	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8090	*/
8091	Log4Func(("TS occurred during event delivery\n"));
8092	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8093	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8094	}
8095
8096	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8097	* emulation. */
8098	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8099	return VERR_EM_INTERPRETER;
8100	}
8101
8102
8103	/**
8104	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8105	*/
8106	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8107	{
8108	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8109	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8110
8111	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8112	if (pVM->hm.s.fTprPatchingAllowed)
8113	{
8114	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8115	if (rc != VERR_NOT_FOUND)
8116	{
8117	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8118	return rc;
8119	}
8120	}
8121
8122	if (EMAreHypercallInstructionsEnabled(pVCpu))
8123	{
8124	unsigned cbInstr;
8125	if (hmR0SvmSupportsNextRipSave(pVCpu))
8126	{
8127	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8128	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8129	}
8130	else
8131	{
8132	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8133	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8134	if ( rc == VINF_SUCCESS
8135	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8136	Assert(cbInstr > 0);
8137	else
8138	cbInstr = 0;
8139	}
8140
8141	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8142	if (RT_SUCCESS(rcStrict))
8143	{
8144	/* Only update the RIP if we're continuing guest execution and not in the case
8145	of say VINF_GIM_R3_HYPERCALL. */
8146	if (rcStrict == VINF_SUCCESS)
8147	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8148
8149	return VBOXSTRICTRC_VAL(rcStrict);
8150	}
8151	else
8152	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8153	}
8154
8155	hmR0SvmSetPendingXcptUD(pVCpu);
8156	return VINF_SUCCESS;
8157	}
8158
8159
8160	/**
8161	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8162	*/
8163	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8164	{
8165	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8166
8167	unsigned cbInstr;
8168	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8169	if (fSupportsNextRipSave)
8170	{
8171	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8172	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8173	}
8174	else
8175	{
8176	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8177	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8178	if ( rc == VINF_SUCCESS
8179	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8180	Assert(cbInstr > 0);
8181	else
8182	cbInstr = 0;
8183	}
8184
8185	/** @todo The guest has likely hit a contended spinlock. We might want to
8186	* poke a schedule different guest VCPU. */
8187	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8188	return VINF_EM_RAW_INTERRUPT;
8189	}
8190
8191
8192	/**
8193	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8194	* \#VMEXIT.
8195	*/
8196	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8197	{
8198	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8199	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8200	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8201
8202	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8203	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8204	}
8205
8206
8207	/**
8208	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8209	*/
8210	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8211	{
8212	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8213
8214	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8215	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8216	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8217
8218	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8219	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8220	* handler executes. Yes, x86 is ugly.
8221	*/
8222	return VINF_EM_RAW_EMULATE_INSTR;
8223	}
8224
8225
8226	/**
8227	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8228	* Conditional \#VMEXIT.
8229	*/
8230	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8231	{
8232	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8233	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8234	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8235
8236	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8237	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8238	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8239	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8240	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8241	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8242
8243	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8244	if (pVM->hmr0.s.fNestedPaging)
8245	{
8246	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8247	if ( !pSvmTransient->fVectoringDoublePF
8248	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8249	{
8250	/* A genuine guest #PF, reflect it to the guest. */
8251	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8252	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8253	uFaultAddress, uErrCode));
8254	}
8255	else
8256	{
8257	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8258	hmR0SvmSetPendingXcptDF(pVCpu);
8259	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8260	}
8261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8262	return VINF_SUCCESS;
8263	}
8264	#endif
8265
8266	Assert(!pVM->hmr0.s.fNestedPaging);
8267
8268	/*
8269	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8270	*/
8271	if ( pVM->hm.s.fTprPatchingAllowed
8272	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8273	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8274	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8275	&& !CPUMIsGuestInLongModeEx(pCtx)
8276	&& !CPUMGetGuestCPL(pVCpu)
8277	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8278	{
8279	RTGCPHYS GCPhysApicBase;
8280	GCPhysApicBase = PDMApicGetBaseMsrNoCheck(pVCpu);
8281	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8282
8283	/* Check if the page at the fault-address is the APIC base. */
8284	PGMPTWALK Walk;
8285	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8286	if ( rc2 == VINF_SUCCESS
8287	&& Walk.GCPhys == GCPhysApicBase)
8288	{
8289	/* Only attempt to patch the instruction once. */
8290	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8291	if (!pPatch)
8292	return VINF_EM_HM_PATCH_TPR_INSTR;
8293	}
8294	}
8295
8296	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8297	pCtx->rip, uErrCode, pCtx->cr3));
8298
8299	/*
8300	* If it's a vectoring #PF, emulate injecting the original event injection as
8301	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8302	* event injection that caused a #PF. See @bugref{6607}.
8303	*/
8304	if (pSvmTransient->fVectoringPF)
8305	{
8306	Assert(pVCpu->hm.s.Event.fPending);
8307	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8308	}
8309
8310	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8311	int rc = PGMTrap0eHandler(pVCpu, uErrCode, pCtx, (RTGCPTR)uFaultAddress);
8312
8313	Log4Func(("#PF: rc=%Rrc\n", rc));
8314
8315	if (rc == VINF_SUCCESS)
8316	{
8317	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8318	TRPMResetTrap(pVCpu);
8319	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8320	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8321	return rc;
8322	}
8323
8324	if (rc == VINF_EM_RAW_GUEST_TRAP)
8325	{
8326	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8327
8328	/*
8329	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8330	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8331	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8332	*/
8333	if ( !pSvmTransient->fVectoringDoublePF
8334	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8335	{
8336	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8337	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8338	TRPMResetTrap(pVCpu);
8339
8340	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8341	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8342	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8343	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8344	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8345	#endif
8346
8347	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8348	}
8349	else
8350	{
8351	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8352	TRPMResetTrap(pVCpu);
8353	hmR0SvmSetPendingXcptDF(pVCpu);
8354	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8355	}
8356
8357	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8358	return VINF_SUCCESS;
8359	}
8360
8361	TRPMResetTrap(pVCpu);
8362	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8363	return rc;
8364	}
8365
8366
8367
8368	/**
8369	* \#VMEXIT handler for division overflow exceptions (SVM_EXIT_XCPT_1).
8370	* Conditional \#VMEXIT.
8371	*/
8372	HMSVM_EXIT_DECL hmR0SvmExitXcptDE(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8373	{
8374	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8375	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8376	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8377
8378	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8379	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8380	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8381
8382	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8383	if (pVCpu->hm.s.fGCMTrapXcptDE)
8384	{
8385	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8386	rc = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx);
8387	AssertMsg(rc == VINF_SUCCESS /* restart / \|\| rc == VERR_NOT_FOUND / deliver exception */, ("rc=%Rrc\n", rc));
8388	}
8389
8390	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
8391	if (RT_FAILURE(rc))
8392	hmR0SvmSetPendingXcptDE(pVCpu);
8393
8394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8395	return VINF_SUCCESS;
8396	}
8397
8398
8399	/**
8400	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8401	* Conditional \#VMEXIT.
8402	*/
8403	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8404	{
8405	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8406	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8408
8409	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8410	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8411	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8412
8413	/** @todo if we accumulate more optional stuff here, we ought to combine the
8414	* reading of opcode bytes to avoid doing more than once. */
8415
8416	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8417	if (pVCpu->hm.s.fGIMTrapXcptUD)
8418	{
8419	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8420	uint8_t cbInstr = 0;
8421	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8422	if (rcStrict == VINF_SUCCESS)
8423	{
8424	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8425	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8426	rcStrict = VINF_SUCCESS;
8427	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8428	}
8429	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8430	rcStrict = VINF_SUCCESS;
8431	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8432	rcStrict = VINF_GIM_R3_HYPERCALL;
8433	else
8434	{
8435	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8436	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8437	}
8438	}
8439
8440	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8441	{
8442	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8443	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8444	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8445	{
8446	/* Ideally, IEM should just handle all these special #UD situations, but
8447	we don't quite trust things to behave optimially when doing that. So,
8448	for now we'll restrict ourselves to a handful of possible sysenter and
8449	sysexit encodings that we filter right here. */
8450	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8451	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8452	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8453	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8454	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8455	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8456	{
8457	cbInstr = cbMin;
8458	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8459	AssertRCStmt(rc2, cbInstr = 0);
8460	}
8461	else
8462	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8463	if ( cbInstr == 0 /* read error */
8464	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8465	\|\| ( uCpl == 0
8466	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8467	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8468	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8469	{
8470	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8471	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8472	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8473	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, GCPtrInstr, abInstr, cbInstr);
8474	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8475	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8476	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8477	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8478	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8479	if (rcStrict == VINF_IEM_RAISED_XCPT)
8480	rcStrict = VINF_SUCCESS;
8481	return rcStrict;
8482	}
8483	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8484	}
8485	else
8486	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8487	}
8488
8489	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8490	if (RT_FAILURE(rcStrict))
8491	{
8492	hmR0SvmSetPendingXcptUD(pVCpu);
8493	rcStrict = VINF_SUCCESS;
8494	}
8495
8496	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8497	return rcStrict;
8498	}
8499
8500
8501	/**
8502	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8503	* Conditional \#VMEXIT.
8504	*/
8505	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8506	{
8507	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8508	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8509	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8510
8511	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8512	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8513
8514	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8515	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8516
8517	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8518
8519	if (!(pCtx->cr0 & X86_CR0_NE))
8520	{
8521	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8522	unsigned cbInstr;
8523	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8524	if (RT_SUCCESS(rc))
8525	{
8526	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8527	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8528	if (RT_SUCCESS(rc))
8529	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8530	}
8531	else
8532	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8533	return rc;
8534	}
8535
8536	hmR0SvmSetPendingXcptMF(pVCpu);
8537	return VINF_SUCCESS;
8538	}
8539
8540
8541	/**
8542	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8543	* \#VMEXIT.
8544	*/
8545	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8546	{
8547	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8548	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8549	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8550	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8551
8552	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8553	{
8554	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8555	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8556	}
8557
8558	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8559
8560	/*
8561	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8562	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8563	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8564	*/
8565	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8566	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8567	int rc = DBGFTrap01Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx, pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8568	if (rc == VINF_EM_RAW_GUEST_TRAP)
8569	{
8570	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8571	if (CPUMIsHyperDebugStateActive(pVCpu))
8572	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8573
8574	/* Reflect the exception back to the guest. */
8575	hmR0SvmSetPendingXcptDB(pVCpu);
8576	rc = VINF_SUCCESS;
8577	}
8578
8579	/*
8580	* Update DR6.
8581	*/
8582	if (CPUMIsHyperDebugStateActive(pVCpu))
8583	{
8584	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8585	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8586	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8587	}
8588	else
8589	{
8590	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8591	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8592	}
8593
8594	return rc;
8595	}
8596
8597
8598	/**
8599	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8600	* Conditional \#VMEXIT.
8601	*/
8602	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8603	{
8604	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8605	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8606	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8607
8608	SVMEVENT Event;
8609	Event.u = 0;
8610	Event.n.u1Valid = 1;
8611	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8612	Event.n.u8Vector = X86_XCPT_AC;
8613	Event.n.u1ErrorCodeValid = 1;
8614	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8615	return VINF_SUCCESS;
8616	}
8617
8618
8619	/**
8620	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8621	* Conditional \#VMEXIT.
8622	*/
8623	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8624	{
8625	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8626	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8627	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8629
8630	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
8631	if (rc == VINF_EM_RAW_GUEST_TRAP)
8632	{
8633	SVMEVENT Event;
8634	Event.u = 0;
8635	Event.n.u1Valid = 1;
8636	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8637	Event.n.u8Vector = X86_XCPT_BP;
8638	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8639	rc = VINF_SUCCESS;
8640	}
8641
8642	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8643	return rc;
8644	}
8645
8646
8647	/**
8648	* Hacks its way around the lovely mesa driver's backdoor accesses.
8649	*
8650	* @sa hmR0VmxHandleMesaDrvGp
8651	*/
8652	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8653	{
8654	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8655	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8656	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8657	RT_NOREF(pCtx, pVmcb);
8658
8659	/* For now we'll just skip the instruction. */
8660	hmR0SvmAdvanceRip(pVCpu, 1);
8661	return VINF_SUCCESS;
8662	}
8663
8664
8665	/**
8666	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8667	* backdoor logging w/o checking what it is running inside.
8668	*
8669	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8670	* backdoor port and magic numbers loaded in registers.
8671	*
8672	* @returns true if it is, false if it isn't.
8673	* @sa hmR0VmxIsMesaDrvGp
8674	*/
8675	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8676	{
8677	/* Check magic and port. */
8678	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8679	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8680	if (pCtx->dx != UINT32_C(0x5658))
8681	return false;
8682	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8683	return false;
8684
8685	/* Check that it is #GP(0). */
8686	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8687	return false;
8688
8689	/* Flat ring-3 CS. */
8690	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8691	if (pVmcb->guest.u8CPL != 3)
8692	return false;
8693	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8694	return false;
8695
8696	/* 0xed: IN eAX,dx */
8697	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8698	{
8699	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8700	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8701	uint8_t abInstr[1];
8702	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8703	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8704	if (RT_FAILURE(rc))
8705	return false;
8706	if (abInstr[0] != 0xed)
8707	return false;
8708	}
8709	else
8710	{
8711	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8712	if (pVmcb->ctrl.abInstr[0] != 0xed)
8713	return false;
8714	}
8715	return true;
8716	}
8717
8718
8719	/**
8720	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8721	* Conditional \#VMEXIT.
8722	*/
8723	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8724	{
8725	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8726	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8727	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8728
8729	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8730	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8731
8732	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8733	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8734	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8735	{
8736	SVMEVENT Event;
8737	Event.u = 0;
8738	Event.n.u1Valid = 1;
8739	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8740	Event.n.u8Vector = X86_XCPT_GP;
8741	Event.n.u1ErrorCodeValid = 1;
8742	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8743	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8744	return VINF_SUCCESS;
8745	}
8746	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8747	}
8748
8749
8750	/**
8751	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8752	*/
8753	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8754	{
8755	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8756	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8757
8758	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8759	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8760	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8761	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8762	Assert(uVector <= X86_XCPT_LAST);
8763	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8764
8765	SVMEVENT Event;
8766	Event.u = 0;
8767	Event.n.u1Valid = 1;
8768	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8769	Event.n.u8Vector = uVector;
8770	switch (uVector)
8771	{
8772	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8773	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8774	case X86_XCPT_DF:
8775	case X86_XCPT_TS:
8776	case X86_XCPT_NP:
8777	case X86_XCPT_SS:
8778	case X86_XCPT_GP:
8779	case X86_XCPT_AC:
8780	{
8781	Event.n.u1ErrorCodeValid = 1;
8782	Event.n.u32ErrorCode = uErrCode;
8783	break;
8784	}
8785	}
8786
8787	#ifdef VBOX_WITH_STATISTICS
8788	switch (uVector)
8789	{
8790	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8791	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8792	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8793	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8794	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8795	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8796	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8797	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8798	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8799	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8800	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8801	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8802	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8803	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8804	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8805	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8806	default:
8807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8808	break;
8809	}
8810	#endif
8811
8812	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8813	return VINF_SUCCESS;
8814	}
8815
8816
8817	/**
8818	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8819	*/
8820	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8821	{
8822	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8823	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8824
8825	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8826	SVMEVENT Event;
8827	Event.u = 0;
8828	Event.n.u1Valid = 1;
8829	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8830	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8831	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8832	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8833	return VINF_SUCCESS;
8834	}
8835
8836
8837	/**
8838	* Generic exit handler that interprets the current instruction
8839	*
8840	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8841	* the exit logging, and this function does the rest.
8842	*/
8843	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8844	uint64_t fExtraImport, uint64_t fHmChanged)
8845	{
8846	#if 1
8847	RT_NOREF(pSvmTransient);
8848	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8849	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8850	if (rcStrict == VINF_SUCCESS)
8851	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8852	else
8853	{
8854	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8855	if (rcStrict == VINF_IEM_RAISED_XCPT)
8856	{
8857	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8858	rcStrict = VINF_SUCCESS;
8859	}
8860	else
8861	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8862	}
8863	return rcStrict;
8864	#else
8865	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8866	return VINF_EM_RAW_EMULATE_INSTR;
8867	#endif
8868	}
8869
8870
8871	/**
8872	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8873	*/
8874	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8875	{
8876	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8877	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8878	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8879	}
8880
8881
8882	/**
8883	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8884	*/
8885	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8886	{
8887	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8888
8889	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8890	mode after a real mode call (like a BIOS call). See ticketref:20625
8891	comment 14. */
8892	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8893	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8894	{
8895	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8896	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8897	}
8898	else
8899	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8900
8901	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8902	}
8903
8904
8905	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8906	/**
8907	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8908	*/
8909	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8910	{
8911	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8912
8913	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8914	Assert(pVmcb);
8915	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8916
8917	VBOXSTRICTRC rcStrict;
8918	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8919	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8920	if (fSupportsNextRipSave)
8921	{
8922	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8923	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8924	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8925	}
8926	else
8927	{
8928	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8929	rcStrict = IEMExecOne(pVCpu);
8930	}
8931
8932	if (rcStrict == VINF_SUCCESS)
8933	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8934	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8935	{
8936	rcStrict = VINF_SUCCESS;
8937	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8938	}
8939	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8940	return rcStrict;
8941	}
8942
8943
8944	/**
8945	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8946	*/
8947	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8948	{
8949	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8950
8951	/*
8952	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8953	* we only intercept STGI when events are pending for GIF to become 1.
8954	*/
8955	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8956	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8957	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8958
8959	VBOXSTRICTRC rcStrict;
8960	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8961	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8962	if (fSupportsNextRipSave)
8963	{
8964	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8965	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8966	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8967	}
8968	else
8969	{
8970	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8971	rcStrict = IEMExecOne(pVCpu);
8972	}
8973
8974	if (rcStrict == VINF_SUCCESS)
8975	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8976	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8977	{
8978	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8979	rcStrict = VINF_SUCCESS;
8980	}
8981	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8982	return rcStrict;
8983	}
8984
8985
8986	/**
8987	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8988	*/
8989	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8990	{
8991	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8992
8993	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8994	Assert(pVmcb);
8995	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8996
8997	VBOXSTRICTRC rcStrict;
8998	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8999	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
9000	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
9001	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
9002	if (fSupportsNextRipSave)
9003	{
9004	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
9005	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9006	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
9007	}
9008	else
9009	{
9010	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9011	rcStrict = IEMExecOne(pVCpu);
9012	}
9013
9014	if (rcStrict == VINF_SUCCESS)
9015	{
9016	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
9017	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
9018	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
9019	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
9020	}
9021	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9022	{
9023	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9024	rcStrict = VINF_SUCCESS;
9025	}
9026	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9027	return rcStrict;
9028	}
9029
9030
9031	/**
9032	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9033	*/
9034	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9035	{
9036	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9037
9038	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9039	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9040
9041	VBOXSTRICTRC rcStrict;
9042	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9043	if (fSupportsNextRipSave)
9044	{
9045	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9046	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9047	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9048	}
9049	else
9050	{
9051	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9052	rcStrict = IEMExecOne(pVCpu);
9053	}
9054
9055	if (rcStrict == VINF_IEM_RAISED_XCPT)
9056	{
9057	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9058	rcStrict = VINF_SUCCESS;
9059	}
9060	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9061	return rcStrict;
9062	}
9063
9064
9065	/**
9066	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9067	*/
9068	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9069	{
9070	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9071
9072	VBOXSTRICTRC rcStrict;
9073	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9074	if (fSupportsNextRipSave)
9075	{
9076	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9077	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9078	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9079	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9080	}
9081	else
9082	{
9083	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9084	rcStrict = IEMExecOne(pVCpu);
9085	}
9086
9087	if (rcStrict == VINF_IEM_RAISED_XCPT)
9088	{
9089	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9090	rcStrict = VINF_SUCCESS;
9091	}
9092	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9093	return rcStrict;
9094	}
9095
9096
9097	/**
9098	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9099	*/
9100	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9101	{
9102	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9103	/* We shall import the entire state here, just in case we enter and continue execution of
9104	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9105	could lose lose part of CPU state. */
9106	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9107
9108	VBOXSTRICTRC rcStrict;
9109	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9110	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9111	if (fSupportsNextRipSave)
9112	{
9113	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9114	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9115	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9116	}
9117	else
9118	{
9119	/* We use IEMExecOneBypassEx() here as it suppresses attempt to continue emulating any
9120	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9121	instruction itself, see @bugref{7243#c126} */
9122	rcStrict = IEMExecOneBypassEx(pVCpu, NULL /* pcbWritten */);
9123	}
9124	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9125
9126	if (rcStrict == VINF_SUCCESS)
9127	{
9128	rcStrict = VINF_SVM_VMRUN;
9129	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9130	}
9131	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9132	{
9133	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9134	rcStrict = VINF_SUCCESS;
9135	}
9136	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9137	return rcStrict;
9138	}
9139
9140
9141	/**
9142	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9143	* Unconditional \#VMEXIT.
9144	*/
9145	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9146	{
9147	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9148	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9149
9150	if (pVCpu->hm.s.Event.fPending)
9151	{
9152	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9153	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9154	}
9155
9156	hmR0SvmSetPendingXcptDB(pVCpu);
9157	return VINF_SUCCESS;
9158	}
9159
9160
9161	/**
9162	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9163	* Conditional \#VMEXIT.
9164	*/
9165	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9166	{
9167	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9168	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9169
9170	SVMEVENT Event;
9171	Event.u = 0;
9172	Event.n.u1Valid = 1;
9173	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9174	Event.n.u8Vector = X86_XCPT_BP;
9175	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9176	return VINF_SUCCESS;
9177	}
9178	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9179
9180	/** @} */
9181

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 107861

Download in other formats: