HMSVMR0.cpp@ 99725

Last change on this file since 99725 was 99211, checked in by vboxsync, 21 months ago
Disassembler,VMM,Runtime: Get rid of deprecated DISCPUSTATE types (preparation for architecture specific separation in order to support ARMv8), bugref:10394 [build fix, missing file]
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.4 KB

Line
1	/* $Id: HMSVMR0.cpp 99211 2023-03-29 14:19:05Z vboxsync $ */
2	/** @file
3	* HM SVM (AMD-V) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2013-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.virtualbox.org.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_HM
33	#define VMCPU_INCL_CPUM_GST_CTX
34	#include <iprt/asm-amd64-x86.h>
35	#include <iprt/thread.h>
36
37	#include <VBox/vmm/pdmapi.h>
38	#include <VBox/vmm/dbgf.h>
39	#include <VBox/vmm/iem.h>
40	#include <VBox/vmm/iom.h>
41	#include <VBox/vmm/tm.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/gcm.h>
44	#include <VBox/vmm/gim.h>
45	#include <VBox/vmm/apic.h>
46	#include "HMInternal.h"
47	#include <VBox/vmm/vmcc.h>
48	#include <VBox/err.h>
49	#include "HMSVMR0.h"
50	#include "dtrace/VBoxVMM.h"
51
52	#ifdef DEBUG_ramshankar
53	# define HMSVM_SYNC_FULL_GUEST_STATE
54	# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
55	# define HMSVM_ALWAYS_TRAP_PF
56	# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
57	#endif
58
59
60	/*********************************************************************************************************************************
61	* Defined Constants And Macros *
62	*********************************************************************************************************************************/
63	#ifdef VBOX_WITH_STATISTICS
64	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
65	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
66	if ((u64ExitCode) == SVM_EXIT_NPF) \
67	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
68	else \
69	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
70	} while (0)
71
72	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
73	STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
74	if ((u64ExitCode) == SVM_EXIT_NPF) \
75	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
76	else \
77	STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
78	} while (0)
79
80	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
81	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
82	if ((u64ExitCode) == SVM_EXIT_NPF) \
83	STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
84	else \
85	STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
86	} while (0)
87	#else
88	# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
89	# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
90	# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
91	#endif /* !VBOX_WITH_STATISTICS */
92
93	/** If we decide to use a function table approach this can be useful to
94	* switch to a "static DECLCALLBACK(int)". */
95	#define HMSVM_EXIT_DECL static VBOXSTRICTRC
96
97	/**
98	* Subset of the guest-CPU state that is kept by SVM R0 code while executing the
99	* guest using hardware-assisted SVM.
100	*
101	* This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
102	* are swapped and restored across the world-switch and also registers like
103	* EFER, PAT MSR etc. which cannot be modified by the guest without causing a
104	* \#VMEXIT.
105	*/
106	#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
107	\| CPUMCTX_EXTRN_RFLAGS \
108	\| CPUMCTX_EXTRN_RAX \
109	\| CPUMCTX_EXTRN_RSP \
110	\| CPUMCTX_EXTRN_SREG_MASK \
111	\| CPUMCTX_EXTRN_CR0 \
112	\| CPUMCTX_EXTRN_CR2 \
113	\| CPUMCTX_EXTRN_CR3 \
114	\| CPUMCTX_EXTRN_TABLE_MASK \
115	\| CPUMCTX_EXTRN_DR6 \
116	\| CPUMCTX_EXTRN_DR7 \
117	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
118	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
119	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
120	\| CPUMCTX_EXTRN_HWVIRT \
121	\| CPUMCTX_EXTRN_INHIBIT_INT \
122	\| CPUMCTX_EXTRN_HM_SVM_MASK)
123
124	/**
125	* Subset of the guest-CPU state that is shared between the guest and host.
126	*/
127	#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
128
129	/** Macro for importing guest state from the VMCB back into CPUMCTX. */
130	#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
131	do { \
132	if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
133	hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
134	} while (0)
135
136	/** Assert that the required state bits are fetched. */
137	#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
138	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
139	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
140
141	/** Assert that preemption is disabled or covered by thread-context hooks. */
142	#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
143	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
144
145	/** Assert that we haven't migrated CPUs when thread-context hooks are not
146	* used. */
147	#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
148	\|\| (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
149	("Illegal migration! Entered on CPU %u Current %u\n", \
150	(a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
151
152	/** Assert that we're not executing a nested-guest. */
153	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
154	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
155	#else
156	# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
157	#endif
158
159	/** Assert that we're executing a nested-guest. */
160	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
161	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
162	#else
163	# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
164	#endif
165
166	/** Macro for checking and returning from the using function for
167	* \#VMEXIT intercepts that maybe caused during delivering of another
168	* event in the guest. */
169	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
170	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
171	do \
172	{ \
173	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
174	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
175	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
176	else if ( rc == VINF_EM_RESET \
177	&& CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
178	{ \
179	HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
180	return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
181	} \
182	else \
183	return rc; \
184	} while (0)
185	#else
186	# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
187	do \
188	{ \
189	int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
190	if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
191	else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
192	else \
193	return rc; \
194	} while (0)
195	#endif
196
197	/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
198	* instruction that exited. */
199	#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
200	do { \
201	if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
202	(a_rc) = VINF_EM_DBG_STEPPED; \
203	} while (0)
204
205	/** Validate segment descriptor granularity bit. */
206	#ifdef VBOX_STRICT
207	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
208	AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
209	\|\| ( (a_pCtx)->reg.Attr.n.u1Granularity \
210	? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
211	: (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
212	("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
213	(a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
214	#else
215	# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
216	#endif
217
218	/**
219	* Exception bitmap mask for all contributory exceptions.
220	*
221	* Page fault is deliberately excluded here as it's conditional as to whether
222	* it's contributory or benign. Page faults are handled separately.
223	*/
224	#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
225	\| RT_BIT(X86_XCPT_DE))
226
227	/**
228	* Mandatory/unconditional guest control intercepts.
229	*
230	* SMIs can and do happen in normal operation. We need not intercept them
231	* while executing the guest (or nested-guest).
232	*/
233	#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
234	\| SVM_CTRL_INTERCEPT_NMI \
235	\| SVM_CTRL_INTERCEPT_INIT \
236	\| SVM_CTRL_INTERCEPT_RDPMC \
237	\| SVM_CTRL_INTERCEPT_CPUID \
238	\| SVM_CTRL_INTERCEPT_RSM \
239	\| SVM_CTRL_INTERCEPT_HLT \
240	\| SVM_CTRL_INTERCEPT_IOIO_PROT \
241	\| SVM_CTRL_INTERCEPT_MSR_PROT \
242	\| SVM_CTRL_INTERCEPT_INVLPGA \
243	\| SVM_CTRL_INTERCEPT_SHUTDOWN \
244	\| SVM_CTRL_INTERCEPT_FERR_FREEZE \
245	\| SVM_CTRL_INTERCEPT_VMRUN \
246	\| SVM_CTRL_INTERCEPT_SKINIT \
247	\| SVM_CTRL_INTERCEPT_WBINVD \
248	\| SVM_CTRL_INTERCEPT_MONITOR \
249	\| SVM_CTRL_INTERCEPT_MWAIT \
250	\| SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
251	\| SVM_CTRL_INTERCEPT_XSETBV)
252
253	/** @name VMCB Clean Bits.
254	*
255	* These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
256	* AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
257	* memory.
258	*
259	* @{ */
260	/** All intercepts vectors, TSC offset, PAUSE filter counter. */
261	#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
262	/** I/O permission bitmap, MSR permission bitmap. */
263	#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
264	/** ASID. */
265	#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
266	/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
267	V_INTR_VECTOR. */
268	#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
269	/** Nested Paging: Nested CR3 (nCR3), PAT. */
270	#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
271	/** Control registers (CR0, CR3, CR4, EFER). */
272	#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
273	/** Debug registers (DR6, DR7). */
274	#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
275	/** GDT, IDT limit and base. */
276	#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
277	/** Segment register: CS, SS, DS, ES limit and base. */
278	#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
279	/** CR2.*/
280	#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
281	/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
282	#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
283	/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
284	PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
285	#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
286	/** Mask of all valid VMCB Clean bits. */
287	#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
288	\| HMSVM_VMCB_CLEAN_IOPM_MSRPM \
289	\| HMSVM_VMCB_CLEAN_ASID \
290	\| HMSVM_VMCB_CLEAN_INT_CTRL \
291	\| HMSVM_VMCB_CLEAN_NP \
292	\| HMSVM_VMCB_CLEAN_CRX_EFER \
293	\| HMSVM_VMCB_CLEAN_DRX \
294	\| HMSVM_VMCB_CLEAN_DT \
295	\| HMSVM_VMCB_CLEAN_SEG \
296	\| HMSVM_VMCB_CLEAN_CR2 \
297	\| HMSVM_VMCB_CLEAN_LBR \
298	\| HMSVM_VMCB_CLEAN_AVIC)
299	/** @} */
300
301	/**
302	* MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
303	*/
304	typedef enum SVMMSREXITREAD
305	{
306	/** Reading this MSR causes a \#VMEXIT. */
307	SVMMSREXIT_INTERCEPT_READ = 0xb,
308	/** Reading this MSR does not cause a \#VMEXIT. */
309	SVMMSREXIT_PASSTHRU_READ
310	} SVMMSREXITREAD;
311
312	/**
313	* MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
314	*/
315	typedef enum SVMMSREXITWRITE
316	{
317	/** Writing to this MSR causes a \#VMEXIT. */
318	SVMMSREXIT_INTERCEPT_WRITE = 0xd,
319	/** Writing to this MSR does not cause a \#VMEXIT. */
320	SVMMSREXIT_PASSTHRU_WRITE
321	} SVMMSREXITWRITE;
322
323	/**
324	* SVM \#VMEXIT handler.
325	*
326	* @returns Strict VBox status code.
327	* @param pVCpu The cross context virtual CPU structure.
328	* @param pSvmTransient Pointer to the SVM-transient structure.
329	*/
330	typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
331
332
333	/*********************************************************************************************************************************
334	* Internal Functions *
335	*********************************************************************************************************************************/
336	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
337	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
338
339
340	/** @name \#VMEXIT handlers.
341	* @{
342	*/
343	static FNSVMEXITHANDLER hmR0SvmExitIntr;
344	static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
345	static FNSVMEXITHANDLER hmR0SvmExitInvd;
346	static FNSVMEXITHANDLER hmR0SvmExitCpuid;
347	static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
348	static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
349	static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
350	static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
351	static FNSVMEXITHANDLER hmR0SvmExitHlt;
352	static FNSVMEXITHANDLER hmR0SvmExitMonitor;
353	static FNSVMEXITHANDLER hmR0SvmExitMwait;
354	static FNSVMEXITHANDLER hmR0SvmExitShutdown;
355	static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
356	static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
357	static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
358	static FNSVMEXITHANDLER hmR0SvmExitMsr;
359	static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
360	static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
361	static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
362	static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
363	static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
364	static FNSVMEXITHANDLER hmR0SvmExitVIntr;
365	static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
366	static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
367	static FNSVMEXITHANDLER hmR0SvmExitPause;
368	static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
369	static FNSVMEXITHANDLER hmR0SvmExitIret;
370	static FNSVMEXITHANDLER hmR0SvmExitXcptDE;
371	static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
372	static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
373	static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
374	static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
375	static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
376	static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
377	static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
378	static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
379	static FNSVMEXITHANDLER hmR0SvmExitSwInt;
380	static FNSVMEXITHANDLER hmR0SvmExitTrRead;
381	static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
382	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
383	static FNSVMEXITHANDLER hmR0SvmExitClgi;
384	static FNSVMEXITHANDLER hmR0SvmExitStgi;
385	static FNSVMEXITHANDLER hmR0SvmExitVmload;
386	static FNSVMEXITHANDLER hmR0SvmExitVmsave;
387	static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
388	static FNSVMEXITHANDLER hmR0SvmExitVmrun;
389	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
390	static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
391	#endif
392	/** @} */
393
394	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
395	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
396	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
397	#endif
398	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
399
400
401	/*********************************************************************************************************************************
402	* Global Variables *
403	*********************************************************************************************************************************/
404	/** Ring-0 memory object for the IO bitmap. */
405	static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
406	/** Physical address of the IO bitmap. */
407	static RTHCPHYS g_HCPhysIOBitmap;
408	/** Pointer to the IO bitmap. */
409	static R0PTRTYPE(void *) g_pvIOBitmap;
410
411	#ifdef VBOX_STRICT
412	# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
413	# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
414	# define HMSVM_LOG_CS RT_BIT_32(2)
415	# define HMSVM_LOG_SS RT_BIT_32(3)
416	# define HMSVM_LOG_FS RT_BIT_32(4)
417	# define HMSVM_LOG_GS RT_BIT_32(5)
418	# define HMSVM_LOG_LBR RT_BIT_32(6)
419	# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
420	\| HMSVM_LOG_CR_REGS \
421	\| HMSVM_LOG_CS \
422	\| HMSVM_LOG_SS \
423	\| HMSVM_LOG_FS \
424	\| HMSVM_LOG_GS \
425	\| HMSVM_LOG_LBR)
426
427	/**
428	* Dumps virtual CPU state and additional info. to the logger for diagnostics.
429	*
430	* @param pVCpu The cross context virtual CPU structure.
431	* @param pVmcb Pointer to the VM control block.
432	* @param pszPrefix Log prefix.
433	* @param fFlags Log flags, see HMSVM_LOG_XXX.
434	* @param uVerbose The verbosity level, currently unused.
435	*/
436	static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
437	{
438	RT_NOREF2(pVCpu, uVerbose);
439	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
440
441	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
442	Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
443
444	if (fFlags & HMSVM_LOG_RBP_RSP)
445	{
446	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RBP);
447	Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
448	}
449
450	if (fFlags & HMSVM_LOG_CR_REGS)
451	{
452	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4);
453	Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
454	}
455
456	if (fFlags & HMSVM_LOG_CS)
457	{
458	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
459	Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
460	pCtx->cs.u32Limit, pCtx->cs.Attr.u));
461	}
462	if (fFlags & HMSVM_LOG_SS)
463	{
464	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
465	Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
466	pCtx->ss.u32Limit, pCtx->ss.Attr.u));
467	}
468	if (fFlags & HMSVM_LOG_FS)
469	{
470	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
471	Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
472	pCtx->fs.u32Limit, pCtx->fs.Attr.u));
473	}
474	if (fFlags & HMSVM_LOG_GS)
475	{
476	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
477	Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
478	pCtx->gs.u32Limit, pCtx->gs.Attr.u));
479	}
480
481	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
482	if (fFlags & HMSVM_LOG_LBR)
483	{
484	Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
485	pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
486	}
487	NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
488	}
489	#endif /* VBOX_STRICT */
490
491
492	/**
493	* Sets up and activates AMD-V on the current CPU.
494	*
495	* @returns VBox status code.
496	* @param pHostCpu The HM physical-CPU structure.
497	* @param pVM The cross context VM structure. Can be
498	* NULL after a resume!
499	* @param pvCpuPage Pointer to the global CPU page.
500	* @param HCPhysCpuPage Physical address of the global CPU page.
501	* @param fEnabledByHost Whether the host OS has already initialized AMD-V.
502	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
503	* unused).
504	*/
505	VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
506	PCSUPHWVIRTMSRS pHwvirtMsrs)
507	{
508	Assert(!fEnabledByHost);
509	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
510	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
511	Assert(pvCpuPage); NOREF(pvCpuPage);
512	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
513
514	RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
515
516	/* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
517	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
518
519	/*
520	* We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
521	*/
522	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
523	if (u64HostEfer & MSR_K6_EFER_SVME)
524	{
525	/* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
526	if ( pVM
527	&& pVM->hm.s.svm.fIgnoreInUseError)
528	pHostCpu->fIgnoreAMDVInUseError = true;
529
530	if (!pHostCpu->fIgnoreAMDVInUseError)
531	{
532	ASMSetFlags(fEFlags);
533	return VERR_SVM_IN_USE;
534	}
535	}
536
537	/* Turn on AMD-V in the EFER MSR. */
538	ASMWrMsr(MSR_K6_EFER, u64HostEfer \| MSR_K6_EFER_SVME);
539
540	/* Write the physical page address where the CPU will store the host state while executing the VM. */
541	ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
542
543	/* Restore interrupts. */
544	ASMSetFlags(fEFlags);
545
546	/*
547	* Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
548	* non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
549	* ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
550	* entirely with before executing any guest code.
551	*/
552	pHostCpu->fFlushAsidBeforeUse = true;
553
554	/*
555	* Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
556	*/
557	++pHostCpu->cTlbFlushes;
558
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Deactivates AMD-V on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pHostCpu The HM physical-CPU structure.
568	* @param pvCpuPage Pointer to the global CPU page.
569	* @param HCPhysCpuPage Physical address of the global CPU page.
570	*/
571	VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
572	{
573	RT_NOREF1(pHostCpu);
574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
575	AssertReturn( HCPhysCpuPage
576	&& HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
580	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
581
582	/* Turn off AMD-V in the EFER MSR. */
583	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
584	ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
585
586	/* Invalidate host state physical address. */
587	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
588
589	/* Restore interrupts. */
590	ASMSetFlags(fEFlags);
591
592	return VINF_SUCCESS;
593	}
594
595
596	/**
597	* Does global AMD-V initialization (called during module initialization).
598	*
599	* @returns VBox status code.
600	*/
601	VMMR0DECL(int) SVMR0GlobalInit(void)
602	{
603	/*
604	* Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
605	* intercept all IO accesses, it's done once globally here instead of per-VM.
606	*/
607	Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
608	int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
609	if (RT_FAILURE(rc))
610	return rc;
611
612	g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
613	g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
614
615	/* Set all bits to intercept all IO accesses. */
616	ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
617
618	return VINF_SUCCESS;
619	}
620
621
622	/**
623	* Does global AMD-V termination (called during module termination).
624	*/
625	VMMR0DECL(void) SVMR0GlobalTerm(void)
626	{
627	if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
628	{
629	RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
630	g_pvIOBitmap = NULL;
631	g_HCPhysIOBitmap = 0;
632	g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
633	}
634	}
635
636
637	/**
638	* Frees any allocated per-VCPU structures for a VM.
639	*
640	* @param pVM The cross context VM structure.
641	*/
642	DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
643	{
644	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
645	{
646	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
647	AssertPtr(pVCpu);
648
649	if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
650	{
651	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
652	pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
653	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
654	}
655
656	if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
657	{
658	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
659	pVCpu->hmr0.s.svm.pVmcb = NULL;
660	pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
661	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
662	}
663
664	if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
665	{
666	RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
667	pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
668	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
669	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
670	}
671	}
672	}
673
674
675	/**
676	* Sets pfnVMRun to the best suited variant.
677	*
678	* This must be called whenever anything changes relative to the SVMR0VMRun
679	* variant selection:
680	* - pVCpu->hm.s.fLoadSaveGuestXcr0
681	* - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
682	* - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
683	* - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
684	* - Perhaps: CPUMCTX.fXStateMask (windows only)
685	*
686	* We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
687	* CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
688	*/
689	static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
690	{
691	static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
692	{
693	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
694	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
695	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
696	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
697	{ hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
698	{ hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
699	{ hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
700	{ hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
701	};
702	uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
703	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
704	\| (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
705	PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
706	if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
707	pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
708	}
709
710
711	/**
712	* Selector FNHMSVMVMRUN implementation.
713	*/
714	static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
715	{
716	hmR0SvmUpdateVmRunFunction(pVCpu);
717	return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
718	}
719
720
721	/**
722	* Does per-VM AMD-V initialization.
723	*
724	* @returns VBox status code.
725	* @param pVM The cross context VM structure.
726	*/
727	VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
728	{
729	int rc = VERR_INTERNAL_ERROR_5;
730
731	/*
732	* Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
733	*/
734	uint32_t u32Family;
735	uint32_t u32Model;
736	uint32_t u32Stepping;
737	if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
738	{
739	Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
740	pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
741	}
742
743	/*
744	* Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
745	*/
746	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
747	{
748	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
749	pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
750	pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
751	pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
752	}
753
754	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
755	{
756	PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
757
758	/*
759	* Initialize the hardware-assisted SVM guest-execution handler.
760	* We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
761	*/
762	pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
763
764	/*
765	* Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
766	* FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
767	*/
768	/** @todo Does this need to be below 4G? */
769	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
770	if (RT_FAILURE(rc))
771	goto failure_cleanup;
772
773	void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
774	pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
775	Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
776	RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
777
778	/*
779	* Allocate one page for the guest-state VMCB.
780	*/
781	/** @todo Does this need to be below 4G? */
782	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
783	if (RT_FAILURE(rc))
784	goto failure_cleanup;
785
786	pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
787	pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
788	Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
789	RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
790
791	/*
792	* Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
793	* SVM to not require one.
794	*/
795	/** @todo Does this need to be below 4G? */
796	rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
797	false /* fExecutable */);
798	if (RT_FAILURE(rc))
799	goto failure_cleanup;
800
801	pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
802	pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
803	/* Set all bits to intercept all MSR accesses (changed later on). */
804	ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
805	}
806
807	return VINF_SUCCESS;
808
809	failure_cleanup:
810	hmR0SvmFreeStructs(pVM);
811	return rc;
812	}
813
814
815	/**
816	* Does per-VM AMD-V termination.
817	*
818	* @returns VBox status code.
819	* @param pVM The cross context VM structure.
820	*/
821	VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
822	{
823	hmR0SvmFreeStructs(pVM);
824	return VINF_SUCCESS;
825	}
826
827
828	/**
829	* Returns whether the VMCB Clean Bits feature is supported.
830	*
831	* @returns @c true if supported, @c false otherwise.
832	* @param pVCpu The cross context virtual CPU structure.
833	* @param fIsNestedGuest Whether we are currently executing the nested-guest.
834	*/
835	DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
836	{
837	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
838	bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
839	if (!fIsNestedGuest)
840	return fHostVmcbCleanBits;
841	return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
842	}
843
844
845	/**
846	* Returns whether the decode assists feature is supported.
847	*
848	* @returns @c true if supported, @c false otherwise.
849	* @param pVCpu The cross context virtual CPU structure.
850	*/
851	DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
852	{
853	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
854	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
855	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
856	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
857	&& pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
858	#endif
859	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
860	}
861
862
863	/**
864	* Returns whether the NRIP_SAVE feature is supported.
865	*
866	* @returns @c true if supported, @c false otherwise.
867	* @param pVCpu The cross context virtual CPU structure.
868	*/
869	DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
870	{
871	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
872	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
873	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
874	return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
875	&& pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
876	#endif
877	return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
878	}
879
880
881	/**
882	* Sets the permission bits for the specified MSR in the MSRPM bitmap.
883	*
884	* @param pVCpu The cross context virtual CPU structure.
885	* @param pbMsrBitmap Pointer to the MSR bitmap.
886	* @param idMsr The MSR for which the permissions are being set.
887	* @param enmRead MSR read permissions.
888	* @param enmWrite MSR write permissions.
889	*
890	* @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
891	* caller needs to take care of this.
892	*/
893	static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
894	SVMMSREXITWRITE enmWrite)
895	{
896	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
897	uint16_t offMsrpm;
898	uint8_t uMsrpmBit;
899	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
900	AssertRC(rc);
901
902	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
903	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
904
905	pbMsrBitmap += offMsrpm;
906	if (enmRead == SVMMSREXIT_INTERCEPT_READ)
907	*pbMsrBitmap \|= RT_BIT(uMsrpmBit);
908	else
909	{
910	if (!fInNestedGuestMode)
911	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
912	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
913	else
914	{
915	/* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
916	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
917	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
918	else
919	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
920	}
921	#endif
922	}
923
924	if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
925	*pbMsrBitmap \|= RT_BIT(uMsrpmBit + 1);
926	else
927	{
928	if (!fInNestedGuestMode)
929	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
930	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
931	else
932	{
933	/* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
934	if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
935	*pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
936	else
937	Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
938	}
939	#endif
940	}
941	}
942
943
944	/**
945	* Sets up AMD-V for the specified VM.
946	* This function is only called once per-VM during initalization.
947	*
948	* @returns VBox status code.
949	* @param pVM The cross context VM structure.
950	*/
951	VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
952	{
953	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
954	AssertReturn(pVM, VERR_INVALID_PARAMETER);
955
956	/*
957	* Validate and copy over some parameters.
958	*/
959	AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
960	bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
961	AssertReturn(!fNestedPaging \|\| (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
962	pVM->hmr0.s.fNestedPaging = fNestedPaging;
963	pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
964
965	/*
966	* Determin some configuration parameters.
967	*/
968	bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
969	bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
970	bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
971
972	bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
973	bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
974
975	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
976	bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
977	bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
978
979	bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
980	bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
981	#endif
982
983	PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
984	PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
985	AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
986	PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
987
988	/* Always trap #AC for reasons of security. */
989	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_AC);
990
991	/* Always trap #DB for reasons of security. */
992	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_DB);
993
994	/* Trap exceptions unconditionally (debug purposes). */
995	#ifdef HMSVM_ALWAYS_TRAP_PF
996	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_PF);
997	#endif
998	#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
999	/* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1000	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT_32(X86_XCPT_BP)
1001	\| RT_BIT_32(X86_XCPT_DE)
1002	\| RT_BIT_32(X86_XCPT_NM)
1003	\| RT_BIT_32(X86_XCPT_UD)
1004	\| RT_BIT_32(X86_XCPT_NP)
1005	\| RT_BIT_32(X86_XCPT_SS)
1006	\| RT_BIT_32(X86_XCPT_GP)
1007	\| RT_BIT_32(X86_XCPT_PF)
1008	\| RT_BIT_32(X86_XCPT_MF)
1009	;
1010	#endif
1011
1012	/* Apply the exceptions intercepts needed by the GIM provider. */
1013	if (pVCpu0->hm.s.fGIMTrapXcptUD \|\| pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1014	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_UD);
1015
1016	/* Apply the exceptions intercepts needed by the GCM fixers. */
1017	if (pVCpu0->hm.s.fGCMTrapXcptDE)
1018	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_DE);
1019
1020	/* The mesa 3d driver hack needs #GP. */
1021	if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1022	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_GP);
1023
1024	/* Set up unconditional intercepts and conditions. */
1025	pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1026	\| SVM_CTRL_INTERCEPT_VMMCALL
1027	\| SVM_CTRL_INTERCEPT_VMSAVE
1028	\| SVM_CTRL_INTERCEPT_VMLOAD
1029	\| SVM_CTRL_INTERCEPT_CLGI
1030	\| SVM_CTRL_INTERCEPT_STGI;
1031
1032	#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1033	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1034	#endif
1035
1036	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1037	if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1038	{
1039	/* Virtualized VMSAVE/VMLOAD. */
1040	if (fUseVirtVmsaveVmload)
1041	{
1042	pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1043	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1044	\| SVM_CTRL_INTERCEPT_VMLOAD);
1045	}
1046	else
1047	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1048
1049	/* Virtual GIF. */
1050	if (fUseVGif)
1051	{
1052	pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1053	pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1054	\| SVM_CTRL_INTERCEPT_STGI);
1055	}
1056	else
1057	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1058	}
1059	else
1060	#endif
1061	{
1062	Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1063	Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1064	Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1065	}
1066
1067	/* CR4 writes must always be intercepted for tracking PGM mode changes and
1068	AVX (for XCR0 syncing during worlds switching). */
1069	pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1070
1071	/* Intercept all DRx reads and writes by default. Changed later on. */
1072	pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1073	pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1074
1075	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1076	pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1077
1078	/* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1079	and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1080	pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1081
1082	/* Set the IO permission bitmap physical addresses. */
1083	pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1084
1085	/* LBR virtualization. */
1086	pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1087
1088	/* The host ASID MBZ, for the guest start with 1. */
1089	pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1090
1091	/* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1092	pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1093
1094	/* Without Nested Paging, we need additionally intercepts. */
1095	if (!fNestedPaging)
1096	{
1097	/* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1098	pVmcbCtrl0->u16InterceptRdCRx \|= RT_BIT(3);
1099	pVmcbCtrl0->u16InterceptWrCRx \|= RT_BIT(3);
1100
1101	/* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1102	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_INVLPG
1103	\| SVM_CTRL_INTERCEPT_TASK_SWITCH;
1104
1105	/* Page faults must be intercepted to implement shadow paging. */
1106	pVmcbCtrl0->u32InterceptXcpt \|= RT_BIT(X86_XCPT_PF);
1107	}
1108
1109	/* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1110	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1111	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_TR_WRITES;
1112
1113	/* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1114	if (fUsePauseFilter)
1115	{
1116	Assert(pVM->hm.s.svm.cPauseFilter > 0);
1117	pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1118	if (fPauseFilterThreshold)
1119	pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1120	pVmcbCtrl0->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_PAUSE;
1121	}
1122
1123	/*
1124	* Setup the MSR permission bitmap.
1125	* The following MSRs are saved/restored automatically during the world-switch.
1126	* Don't intercept guest read/write accesses to these MSRs.
1127	*/
1128	uint8_t pbMsrBitmap0 = (uint8_t )pVCpu0->hmr0.s.svm.pvMsrBitmap;
1129	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1130	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1131	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1132	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1133	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1134	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1135	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1136	if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1137	{
1138	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1139	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1140	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1141	}
1142	else
1143	{
1144	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1145	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1146	hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1147	}
1148	pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1149
1150	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1151	Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1152
1153	for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1154	{
1155	PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1156	PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1157	AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1158	PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1159
1160	/* Copy the VMCB control area. */
1161	memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1162
1163	/* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1164	uint8_t pbMsrBitmapCur = (uint8_t )pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1165	memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1166	pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1167
1168	/* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1169	Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1170
1171	/* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1172	Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1173	/* Same for GCM, #DE trapping should be uniform across VCPUs. */
1174	Assert(pVCpuCur->hm.s.fGCMTrapXcptDE == pVCpu0->hm.s.fGCMTrapXcptDE);
1175	}
1176
1177	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1178	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1179	fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1180	#else
1181	LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1182	#endif
1183	return VINF_SUCCESS;
1184	}
1185
1186
1187	/**
1188	* Gets a pointer to the currently active guest (or nested-guest) VMCB.
1189	*
1190	* @returns Pointer to the current context VMCB.
1191	* @param pVCpu The cross context virtual CPU structure.
1192	*/
1193	DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1194	{
1195	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1196	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1197	return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1198	#endif
1199	return pVCpu->hmr0.s.svm.pVmcb;
1200	}
1201
1202
1203	/**
1204	* Gets a pointer to the nested-guest VMCB cache.
1205	*
1206	* @returns Pointer to the nested-guest VMCB cache.
1207	* @param pVCpu The cross context virtual CPU structure.
1208	*/
1209	DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1210	{
1211	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1212	Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1213	return &pVCpu->hm.s.svm.NstGstVmcbCache;
1214	#else
1215	RT_NOREF(pVCpu);
1216	return NULL;
1217	#endif
1218	}
1219
1220
1221	/**
1222	* Invalidates a guest page by guest virtual address.
1223	*
1224	* @returns VBox status code.
1225	* @param pVCpu The cross context virtual CPU structure.
1226	* @param GCVirt Guest virtual address of the page to invalidate.
1227	*/
1228	VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1229	{
1230	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1231
1232	bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) \|\| pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1233
1234	/* Skip it if a TLB flush is already pending. */
1235	if (!fFlushPending)
1236	{
1237	Log4Func(("%#RGv\n", GCVirt));
1238
1239	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1240	AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1241
1242	SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1243	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1244	}
1245	return VINF_SUCCESS;
1246	}
1247
1248
1249	/**
1250	* Flushes the appropriate tagged-TLB entries.
1251	*
1252	* @param pHostCpu The HM physical-CPU structure.
1253	* @param pVCpu The cross context virtual CPU structure.
1254	* @param pVmcb Pointer to the VM control block.
1255	*/
1256	static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1257	{
1258	/*
1259	* Force a TLB flush for the first world switch if the current CPU differs from the one
1260	* we ran on last. This can happen both for start & resume due to long jumps back to
1261	* ring-3.
1262	*
1263	* We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1264	* correlation between it and the physical CPU.
1265	*
1266	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1267	* flushing the TLB, so we cannot reuse the ASIDs without flushing.
1268	*/
1269	bool fNewAsid = false;
1270	Assert(pHostCpu->idCpu != NIL_RTCPUID);
1271	if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1272	\|\| pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1273	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1274	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1275	#endif
1276	)
1277	{
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1279	pVCpu->hmr0.s.fForceTLBFlush = true;
1280	fNewAsid = true;
1281	}
1282
1283	/* Set TLB flush state as checked until we return from the world switch. */
1284	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1285
1286	/* Check for explicit TLB flushes. */
1287	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1288	{
1289	pVCpu->hmr0.s.fForceTLBFlush = true;
1290	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1291	}
1292
1293	/*
1294	* If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1295	* This Host CPU requirement takes precedence.
1296	*/
1297	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1298	if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1299	{
1300	pHostCpu->uCurrentAsid = 1;
1301	pVCpu->hmr0.s.uCurrentAsid = 1;
1302	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1303	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1304	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1305
1306	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1307	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1308	}
1309	else
1310	{
1311	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1312	if (pVCpu->hmr0.s.fForceTLBFlush)
1313	{
1314	/* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1315	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1316
1317	if (fNewAsid)
1318	{
1319	++pHostCpu->uCurrentAsid;
1320
1321	bool fHitASIDLimit = false;
1322	if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1323	{
1324	pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1325	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1326	fHitASIDLimit = true;
1327	}
1328
1329	if ( fHitASIDLimit
1330	\|\| pHostCpu->fFlushAsidBeforeUse)
1331	{
1332	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1333	pHostCpu->fFlushAsidBeforeUse = false;
1334	}
1335
1336	pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1337	pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1338	pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1339	}
1340	else
1341	{
1342	if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1343	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1344	else
1345	pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1346	}
1347
1348	pVCpu->hmr0.s.fForceTLBFlush = false;
1349	}
1350	}
1351
1352	/* Update VMCB with the ASID. */
1353	if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1354	{
1355	pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1356	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1357	}
1358
1359	AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1360	("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1361	AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1362	("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1363	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1364	("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1365	AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1366	("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1367
1368	#ifdef VBOX_WITH_STATISTICS
1369	if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1370	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1371	else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1372	\|\| pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1373	{
1374	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1375	}
1376	else
1377	{
1378	Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1379	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1380	}
1381	#endif
1382	}
1383
1384
1385	/**
1386	* Sets an exception intercept in the specified VMCB.
1387	*
1388	* @param pVmcb Pointer to the VM control block.
1389	* @param uXcpt The exception (X86_XCPT_*).
1390	*/
1391	DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1392	{
1393	if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1394	{
1395	pVmcb->ctrl.u32InterceptXcpt \|= RT_BIT(uXcpt);
1396	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1397	}
1398	}
1399
1400
1401	/**
1402	* Clears an exception intercept in the specified VMCB.
1403	*
1404	* @param pVCpu The cross context virtual CPU structure.
1405	* @param pVmcb Pointer to the VM control block.
1406	* @param uXcpt The exception (X86_XCPT_*).
1407	*
1408	* @remarks This takes into account if we're executing a nested-guest and only
1409	* removes the exception intercept if both the guest -and- nested-guest
1410	* are not intercepting it.
1411	*/
1412	DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1413	{
1414	Assert(uXcpt != X86_XCPT_DB);
1415	Assert(uXcpt != X86_XCPT_AC);
1416	Assert(uXcpt != X86_XCPT_GP);
1417	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1418	if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1419	{
1420	bool fRemove = true;
1421	# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1422	/* Only remove the intercept if the nested-guest is also not intercepting it! */
1423	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1424	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1425	{
1426	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1427	fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1428	}
1429	# else
1430	RT_NOREF(pVCpu);
1431	# endif
1432	if (fRemove)
1433	{
1434	pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1435	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1436	}
1437	}
1438	#else
1439	RT_NOREF3(pVCpu, pVmcb, uXcpt);
1440	#endif
1441	}
1442
1443
1444	/**
1445	* Sets a control intercept in the specified VMCB.
1446	*
1447	* @param pVmcb Pointer to the VM control block.
1448	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1449	*/
1450	DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1451	{
1452	if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1453	{
1454	pVmcb->ctrl.u64InterceptCtrl \|= fCtrlIntercept;
1455	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1456	}
1457	}
1458
1459
1460	/**
1461	* Clears a control intercept in the specified VMCB.
1462	*
1463	* @returns @c true if the intercept is still set, @c false otherwise.
1464	* @param pVCpu The cross context virtual CPU structure.
1465	* @param pVmcb Pointer to the VM control block.
1466	* @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1467	*
1468	* @remarks This takes into account if we're executing a nested-guest and only
1469	* removes the control intercept if both the guest -and- nested-guest
1470	* are not intercepting it.
1471	*/
1472	static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1473	{
1474	if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1475	{
1476	bool fRemove = true;
1477	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1478	/* Only remove the control intercept if the nested-guest is also not intercepting it! */
1479	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1480	{
1481	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1482	fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1483	}
1484	#else
1485	RT_NOREF(pVCpu);
1486	#endif
1487	if (fRemove)
1488	{
1489	pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1490	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1491	}
1492	}
1493
1494	return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1495	}
1496
1497
1498	/**
1499	* Exports the guest (or nested-guest) CR0 into the VMCB.
1500	*
1501	* @param pVCpu The cross context virtual CPU structure.
1502	* @param pVmcb Pointer to the VM control block.
1503	*
1504	* @remarks This assumes we always pre-load the guest FPU.
1505	* @remarks No-long-jump zone!!!
1506	*/
1507	static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1508	{
1509	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1510
1511	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1512	uint64_t const uGuestCr0 = pCtx->cr0;
1513	uint64_t uShadowCr0 = uGuestCr0;
1514
1515	/* Always enable caching. */
1516	uShadowCr0 &= ~(X86_CR0_CD \| X86_CR0_NW);
1517
1518	/* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1519	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1520	{
1521	uShadowCr0 \|= X86_CR0_PG /* Use shadow page tables. */
1522	\| X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1523	}
1524
1525	/*
1526	* Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1527	* lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1528	* see @bugref{7243#c103}.
1529	*/
1530	if (!(uGuestCr0 & X86_CR0_NE))
1531	{
1532	uShadowCr0 \|= X86_CR0_NE;
1533	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1534	}
1535	else
1536	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1537
1538	/*
1539	* If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1540	*
1541	* CR0 writes still needs interception as PGM requires tracking paging mode changes,
1542	* see @bugref{6944}.
1543	*
1544	* We also don't ever want to honor weird things like cache disable from the guest.
1545	* However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1546	* write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1547	*/
1548	if (uShadowCr0 == uGuestCr0)
1549	{
1550	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1551	{
1552	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1553	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1554	Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1555	}
1556	else
1557	{
1558	/* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1559	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1560	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1561	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1562	pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1563	\| (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1564	}
1565	}
1566	else
1567	{
1568	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(0);
1569	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(0);
1570	}
1571	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1572
1573	Assert(!RT_HI_U32(uShadowCr0));
1574	if (pVmcb->guest.u64CR0 != uShadowCr0)
1575	{
1576	pVmcb->guest.u64CR0 = uShadowCr0;
1577	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1578	}
1579	}
1580
1581
1582	/**
1583	* Exports the guest (or nested-guest) CR3 into the VMCB.
1584	*
1585	* @param pVCpu The cross context virtual CPU structure.
1586	* @param pVmcb Pointer to the VM control block.
1587	*
1588	* @remarks No-long-jump zone!!!
1589	*/
1590	static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1591	{
1592	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593
1594	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1595	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1596	if (pVM->hmr0.s.fNestedPaging)
1597	{
1598	pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1599	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1600	pVmcb->guest.u64CR3 = pCtx->cr3;
1601	Assert(pVmcb->ctrl.u64NestedPagingCR3);
1602	}
1603	else
1604	pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1605
1606	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1607	}
1608
1609
1610	/**
1611	* Exports the guest (or nested-guest) CR4 into the VMCB.
1612	*
1613	* @param pVCpu The cross context virtual CPU structure.
1614	* @param pVmcb Pointer to the VM control block.
1615	*
1616	* @remarks No-long-jump zone!!!
1617	*/
1618	static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1619	{
1620	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1621
1622	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1623	uint64_t uShadowCr4 = pCtx->cr4;
1624	if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1625	{
1626	switch (pVCpu->hm.s.enmShadowMode)
1627	{
1628	case PGMMODE_REAL:
1629	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1630	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1631
1632	case PGMMODE_32_BIT: /* 32-bit paging. */
1633	uShadowCr4 &= ~X86_CR4_PAE;
1634	break;
1635
1636	case PGMMODE_PAE: /* PAE paging. */
1637	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1638	/** Must use PAE paging as we could use physical memory > 4 GB */
1639	uShadowCr4 \|= X86_CR4_PAE;
1640	break;
1641
1642	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1643	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1644	#ifdef VBOX_WITH_64_BITS_GUESTS
1645	break;
1646	#else
1647	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1648	#endif
1649
1650	default: /* shut up gcc */
1651	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1652	}
1653	}
1654
1655	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1656	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1657	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1658	{
1659	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1660	hmR0SvmUpdateVmRunFunction(pVCpu);
1661	}
1662
1663	/* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1664	if (uShadowCr4 == pCtx->cr4)
1665	{
1666	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1667	pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1668	else
1669	{
1670	/* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1671	PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1672	pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1673	\| (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1674	}
1675	}
1676	else
1677	pVmcb->ctrl.u16InterceptRdCRx \|= RT_BIT(4);
1678
1679	/* CR4 writes are always intercepted (both guest, nested-guest) for tracking
1680	PGM mode changes and AVX (for XCR0 syncing during worlds switching). */
1681	Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1682
1683	/* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1684	Assert(!RT_HI_U32(uShadowCr4));
1685	pVmcb->guest.u64CR4 = uShadowCr4;
1686	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER \| HMSVM_VMCB_CLEAN_INTERCEPTS);
1687
1688	return VINF_SUCCESS;
1689	}
1690
1691
1692	/**
1693	* Exports the guest (or nested-guest) control registers into the VMCB.
1694	*
1695	* @returns VBox status code.
1696	* @param pVCpu The cross context virtual CPU structure.
1697	* @param pVmcb Pointer to the VM control block.
1698	*
1699	* @remarks No-long-jump zone!!!
1700	*/
1701	static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1702	{
1703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1704
1705	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1706	{
1707	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1708	hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1709
1710	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1711	{
1712	pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1713	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1714	}
1715
1716	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1717	hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1718
1719	/* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1720	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1721	{
1722	int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1723	if (RT_FAILURE(rc))
1724	return rc;
1725	}
1726
1727	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1728	}
1729	return VINF_SUCCESS;
1730	}
1731
1732
1733	/**
1734	* Exports the guest (or nested-guest) segment registers into the VMCB.
1735	*
1736	* @returns VBox status code.
1737	* @param pVCpu The cross context virtual CPU structure.
1738	* @param pVmcb Pointer to the VM control block.
1739	*
1740	* @remarks No-long-jump zone!!!
1741	*/
1742	static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1743	{
1744	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1745	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1746
1747	/* Guest segment registers. */
1748	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1749	{
1750	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1751	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1752
1753	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1754	{
1755	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1756	pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1757	}
1758
1759	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1760	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1761
1762	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1763	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1764
1765	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1766	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1767
1768	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1769	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1770
1771	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1772	}
1773
1774	/* Guest TR. */
1775	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1776	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1777
1778	/* Guest LDTR. */
1779	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1780	HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1781
1782	/* Guest GDTR. */
1783	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1784	{
1785	pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1786	pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1787	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1788	}
1789
1790	/* Guest IDTR. */
1791	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1792	{
1793	pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1794	pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1795	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1796	}
1797
1798	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1799	\| HM_CHANGED_GUEST_TABLE_MASK);
1800	}
1801
1802
1803	/**
1804	* Exports the guest (or nested-guest) MSRs into the VMCB.
1805	*
1806	* @param pVCpu The cross context virtual CPU structure.
1807	* @param pVmcb Pointer to the VM control block.
1808	*
1809	* @remarks No-long-jump zone!!!
1810	*/
1811	static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1812	{
1813	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1814	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1815
1816	/* Guest Sysenter MSRs. */
1817	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1818	{
1819	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1820	pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1821
1822	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1823	pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1824
1825	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1826	pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1827	}
1828
1829	/*
1830	* Guest EFER MSR.
1831	* AMD-V requires guest EFER.SVME to be set. Weird.
1832	* See AMD spec. 15.5.1 "Basic Operation" \| "Canonicalization and Consistency Checks".
1833	*/
1834	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1835	{
1836	pVmcb->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
1837	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1838	}
1839
1840	/* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1841	if ( !CPUMIsGuestInLongModeEx(pCtx)
1842	&& (pCtx->msrEFER & MSR_K6_EFER_LME))
1843	{
1844	pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1845	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1846	}
1847
1848	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1849	{
1850	pVmcb->guest.u64STAR = pCtx->msrSTAR;
1851	pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1852	pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1853	pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1854	}
1855
1856	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1857	pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1858
1859	pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1860	\| HM_CHANGED_GUEST_EFER_MSR
1861	\| HM_CHANGED_GUEST_SYSCALL_MSRS
1862	\| HM_CHANGED_GUEST_KERNEL_GS_BASE);
1863
1864	/*
1865	* Setup the PAT MSR (applicable for Nested Paging only).
1866	*
1867	* The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1868	* as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1869	*
1870	* While guests can modify and see the modified values through the shadow values,
1871	* we shall not honor any guest modifications of this MSR to ensure caching is always
1872	* enabled similar to how we clear CR0.CD and NW bits.
1873	*
1874	* For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1875	*/
1876	pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1877
1878	/* Enable the last branch record bit if LBR virtualization is enabled. */
1879	if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1880	pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1881	}
1882
1883
1884	/**
1885	* Exports the guest (or nested-guest) debug state into the VMCB and programs
1886	* the necessary intercepts accordingly.
1887	*
1888	* @param pVCpu The cross context virtual CPU structure.
1889	* @param pVmcb Pointer to the VM control block.
1890	*
1891	* @remarks No-long-jump zone!!!
1892	* @remarks Requires EFLAGS to be up-to-date in the VMCB!
1893	*/
1894	static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1895	{
1896	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1897
1898	/** @todo Figure out stepping with nested-guest. */
1899	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1900	{
1901	/*
1902	* We don't want to always intercept DRx read/writes for nested-guests as it causes
1903	* problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1904	* Instead, they are strictly only requested when the nested hypervisor intercepts
1905	* them -- handled while merging VMCB controls.
1906	*
1907	* If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1908	* then the nested-guest debug state should be actively loaded on the host so that
1909	* nested-guest reads/writes its own debug registers without causing VM-exits.
1910	*/
1911	if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1912	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1913	&& !CPUMIsGuestDebugStateActive(pVCpu))
1914	{
1915	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1916	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1917	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1918	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1919	}
1920
1921	pVmcb->guest.u64DR6 = pCtx->dr[6];
1922	pVmcb->guest.u64DR7 = pCtx->dr[7];
1923	return;
1924	}
1925
1926	/*
1927	* Anyone single stepping on the host side? If so, we'll have to use the
1928	* trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1929	* the VMM level like the VT-x implementations does.
1930	*/
1931	bool fInterceptMovDRx = false;
1932	bool const fStepping = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
1933	if (fStepping)
1934	{
1935	pVCpu->hmr0.s.fClearTrapFlag = true;
1936	pVmcb->guest.u64RFlags \|= X86_EFL_TF;
1937	fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1938	}
1939
1940	if ( fStepping
1941	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1942	{
1943	/*
1944	* Use the combined guest and host DRx values found in the hypervisor
1945	* register set because the debugger has breakpoints active or someone
1946	* is single stepping on the host side.
1947	*
1948	* Note! DBGF expects a clean DR6 state before executing guest code.
1949	*/
1950	if (!CPUMIsHyperDebugStateActive(pVCpu))
1951	{
1952	CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1953	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1954	Assert(CPUMIsHyperDebugStateActive(pVCpu));
1955	}
1956
1957	/* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1958	if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1959	\|\| pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1960	{
1961	pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1962	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1963	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1964	}
1965
1966	/** @todo If we cared, we could optimize to allow the guest to read registers
1967	* with the same values. */
1968	fInterceptMovDRx = true;
1969	pVCpu->hmr0.s.fUsingHyperDR7 = true;
1970	Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
1971	}
1972	else
1973	{
1974	/*
1975	* Update DR6, DR7 with the guest values if necessary.
1976	*/
1977	if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1978	\|\| pVmcb->guest.u64DR6 != pCtx->dr[6])
1979	{
1980	pVmcb->guest.u64DR7 = pCtx->dr[7];
1981	pVmcb->guest.u64DR6 = pCtx->dr[6];
1982	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1983	}
1984	pVCpu->hmr0.s.fUsingHyperDR7 = false;
1985
1986	/*
1987	* If the guest has enabled debug registers, we need to load them prior to
1988	* executing guest code so they'll trigger at the right time.
1989	*/
1990	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
1991	{
1992	if (!CPUMIsGuestDebugStateActive(pVCpu))
1993	{
1994	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1995	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1996	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1997	Assert(CPUMIsGuestDebugStateActive(pVCpu));
1998	}
1999	Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2000	}
2001	/*
2002	* If no debugging enabled, we'll lazy load DR0-3. We don't need to
2003	* intercept #DB as DR6 is updated in the VMCB.
2004	*
2005	* Note! If we cared and dared, we could skip intercepting \#DB here.
2006	* However, \#DB shouldn't be performance critical, so we'll play safe
2007	* and keep the code similar to the VT-x code and always intercept it.
2008	*/
2009	else if (!CPUMIsGuestDebugStateActive(pVCpu))
2010	fInterceptMovDRx = true;
2011	}
2012
2013	Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2014	if (fInterceptMovDRx)
2015	{
2016	if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2017	\|\| pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2018	{
2019	pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2020	pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2021	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2022	}
2023	}
2024	else
2025	{
2026	if ( pVmcb->ctrl.u16InterceptRdDRx
2027	\|\| pVmcb->ctrl.u16InterceptWrDRx)
2028	{
2029	pVmcb->ctrl.u16InterceptRdDRx = 0;
2030	pVmcb->ctrl.u16InterceptWrDRx = 0;
2031	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2032	}
2033	}
2034	Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2035	}
2036
2037	/**
2038	* Exports the hardware virtualization state into the nested-guest
2039	* VMCB.
2040	*
2041	* @param pVCpu The cross context virtual CPU structure.
2042	* @param pVmcb Pointer to the VM control block.
2043	*
2044	* @remarks No-long-jump zone!!!
2045	*/
2046	static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2047	{
2048	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2049
2050	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2051	{
2052	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2053	{
2054	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2055	PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2056
2057	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2058	Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2059	Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2060	NOREF(pVM);
2061
2062	pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2063	}
2064
2065	/*
2066	* Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2067	* since SVM doesn't have a preemption timer.
2068	*
2069	* We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2070	* nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2071	* and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2072	*/
2073	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2074	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2075	uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2076	uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2077	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2078	{
2079	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2080	pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2081	pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2082	}
2083	else
2084	{
2085	/** @todo r=ramshankar: We can turn these assignments into assertions. */
2086	pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2087	pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2088	}
2089	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2090
2091	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2092	}
2093	}
2094
2095
2096	/**
2097	* Exports the guest APIC TPR state into the VMCB.
2098	*
2099	* @returns VBox status code.
2100	* @param pVCpu The cross context virtual CPU structure.
2101	* @param pVmcb Pointer to the VM control block.
2102	*/
2103	static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2104	{
2105	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2106
2107	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2108	{
2109	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2110	if ( PDMHasApic(pVM)
2111	&& APICIsEnabled(pVCpu))
2112	{
2113	bool fPendingIntr;
2114	uint8_t u8Tpr;
2115	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2116	AssertRCReturn(rc, rc);
2117
2118	/* Assume that we need to trap all TPR accesses and thus need not check on
2119	every #VMEXIT if we should update the TPR. */
2120	Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2121	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2122
2123	if (!pVM->hm.s.fTprPatchingActive)
2124	{
2125	/* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2126	pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2127
2128	/* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2129	can deliver the interrupt to the guest. */
2130	if (fPendingIntr)
2131	pVmcb->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
2132	else
2133	{
2134	pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2135	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2136	}
2137
2138	pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS \| HMSVM_VMCB_CLEAN_INT_CTRL);
2139	}
2140	else
2141	{
2142	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2143	pVmcb->guest.u64LSTAR = u8Tpr;
2144	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hmr0.s.svm.pvMsrBitmap;
2145
2146	/* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2147	if (fPendingIntr)
2148	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2149	else
2150	{
2151	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2152	pVCpu->hmr0.s.svm.fSyncVTpr = true;
2153	}
2154	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2155	}
2156	}
2157	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2158	}
2159	return VINF_SUCCESS;
2160	}
2161
2162
2163	/**
2164	* Sets up the exception interrupts required for guest execution in the VMCB.
2165	*
2166	* @param pVCpu The cross context virtual CPU structure.
2167	* @param pVmcb Pointer to the VM control block.
2168	*
2169	* @remarks No-long-jump zone!!!
2170	*/
2171	static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2172	{
2173	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2174
2175	/* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2176	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2177	{
2178	/* Trap #UD for GIM provider (e.g. for hypercalls). */
2179	if (pVCpu->hm.s.fGIMTrapXcptUD \|\| pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2180	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2181	else
2182	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2183
2184	/* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2185	if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2186	hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2187	else
2188	hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2189
2190	/* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2191	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2192	}
2193	}
2194
2195
2196	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2197	/**
2198	* Merges guest and nested-guest intercepts for executing the nested-guest using
2199	* hardware-assisted SVM.
2200	*
2201	* This merges the guest and nested-guest intercepts in a way that if the outer
2202	* guest intercept is set we need to intercept it in the nested-guest as
2203	* well.
2204	*
2205	* @param pVCpu The cross context virtual CPU structure.
2206	* @param pVmcbNstGst Pointer to the nested-guest VM control block.
2207	*/
2208	static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2209	{
2210	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2211	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2212	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2213	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2214
2215	/* Merge the guest's CR intercepts into the nested-guest VMCB. */
2216	pVmcbNstGstCtrl->u16InterceptRdCRx \|= pVmcb->ctrl.u16InterceptRdCRx;
2217	pVmcbNstGstCtrl->u16InterceptWrCRx \|= pVmcb->ctrl.u16InterceptWrCRx;
2218
2219	/* Always intercept CR4 writes for tracking PGM mode changes and AVX (for
2220	XCR0 syncing during worlds switching). */
2221	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(4);
2222
2223	/* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2224	if (!pVM->hmr0.s.fNestedPaging)
2225	{
2226	pVmcbNstGstCtrl->u16InterceptRdCRx \|= RT_BIT(3);
2227	pVmcbNstGstCtrl->u16InterceptWrCRx \|= RT_BIT(3);
2228	}
2229
2230	/* Merge the guest's DR intercepts into the nested-guest VMCB. */
2231	pVmcbNstGstCtrl->u16InterceptRdDRx \|= pVmcb->ctrl.u16InterceptRdDRx;
2232	pVmcbNstGstCtrl->u16InterceptWrDRx \|= pVmcb->ctrl.u16InterceptWrDRx;
2233
2234	/*
2235	* Merge the guest's exception intercepts into the nested-guest VMCB.
2236	*
2237	* - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2238	* while executing the nested-guest.
2239	*
2240	* - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2241	* be tweaked later depending on how we wish to implement breakpoints.
2242	*
2243	* - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2244	* loaded into their guests, not ours.
2245	*
2246	* Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2247	* for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2248	*/
2249	#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2250	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt
2251	& ~( RT_BIT(X86_XCPT_UD)
2252	\| RT_BIT(X86_XCPT_BP)
2253	\| (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2254	#else
2255	pVmcbNstGstCtrl->u32InterceptXcpt \|= pVmcb->ctrl.u32InterceptXcpt;
2256	#endif
2257
2258	/*
2259	* Adjust intercepts while executing the nested-guest that differ from the
2260	* outer guest intercepts.
2261	*
2262	* - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2263	* that belong to the nested-guest to the outer guest.
2264	*
2265	* - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2266	* the nested-guest, the physical CPU raises a \#UD exception as expected.
2267	*/
2268	pVmcbNstGstCtrl->u64InterceptCtrl \|= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2269	\| SVM_CTRL_INTERCEPT_VMMCALL))
2270	\| HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2271
2272	Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2273	== HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2274
2275	/* Finally, update the VMCB clean bits. */
2276	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2277	}
2278	#endif
2279
2280
2281	/**
2282	* Enters the AMD-V session.
2283	*
2284	* @returns VBox status code.
2285	* @param pVCpu The cross context virtual CPU structure.
2286	*/
2287	VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2288	{
2289	AssertPtr(pVCpu);
2290	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2292
2293	LogFlowFunc(("pVCpu=%p\n", pVCpu));
2294	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2295	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2296
2297	pVCpu->hmr0.s.fLeaveDone = false;
2298	return VINF_SUCCESS;
2299	}
2300
2301
2302	/**
2303	* Thread-context callback for AMD-V.
2304	*
2305	* This is used together with RTThreadCtxHookCreate() on platforms which
2306	* supports it, and directly from VMMR0EmtPrepareForBlocking() and
2307	* VMMR0EmtResumeAfterBlocking() on platforms which don't.
2308	*
2309	* @param enmEvent The thread-context event.
2310	* @param pVCpu The cross context virtual CPU structure.
2311	* @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2312	* @thread EMT(pVCpu)
2313	*/
2314	VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2315	{
2316	NOREF(fGlobalInit);
2317
2318	switch (enmEvent)
2319	{
2320	case RTTHREADCTXEVENT_OUT:
2321	{
2322	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2323	VMCPU_ASSERT_EMT(pVCpu);
2324
2325	/* No longjmps (log-flush, locks) in this fragile context. */
2326	VMMRZCallRing3Disable(pVCpu);
2327
2328	if (!pVCpu->hmr0.s.fLeaveDone)
2329	{
2330	hmR0SvmLeave(pVCpu, false /* fImportState */);
2331	pVCpu->hmr0.s.fLeaveDone = true;
2332	}
2333
2334	/* Leave HM context, takes care of local init (term). */
2335	int rc = HMR0LeaveCpu(pVCpu);
2336	AssertRC(rc); NOREF(rc);
2337
2338	/* Restore longjmp state. */
2339	VMMRZCallRing3Enable(pVCpu);
2340	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2341	break;
2342	}
2343
2344	case RTTHREADCTXEVENT_IN:
2345	{
2346	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2347	VMCPU_ASSERT_EMT(pVCpu);
2348
2349	/* No longjmps (log-flush, locks) in this fragile context. */
2350	VMMRZCallRing3Disable(pVCpu);
2351
2352	/*
2353	* Initialize the bare minimum state required for HM. This takes care of
2354	* initializing AMD-V if necessary (onlined CPUs, local init etc.)
2355	*/
2356	int rc = hmR0EnterCpu(pVCpu);
2357	AssertRC(rc); NOREF(rc);
2358	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2359	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2360
2361	pVCpu->hmr0.s.fLeaveDone = false;
2362
2363	/* Restore longjmp state. */
2364	VMMRZCallRing3Enable(pVCpu);
2365	break;
2366	}
2367
2368	default:
2369	break;
2370	}
2371	}
2372
2373
2374	/**
2375	* Saves the host state.
2376	*
2377	* @returns VBox status code.
2378	* @param pVCpu The cross context virtual CPU structure.
2379	*
2380	* @remarks No-long-jump zone!!!
2381	*/
2382	VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2383	{
2384	NOREF(pVCpu);
2385
2386	/* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2387	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2388	return VINF_SUCCESS;
2389	}
2390
2391
2392	/**
2393	* Exports the guest or nested-guest state from the virtual-CPU context into the
2394	* VMCB.
2395	*
2396	* Also sets up the appropriate VMRUN function to execute guest or nested-guest
2397	* code based on the virtual-CPU mode.
2398	*
2399	* @returns VBox status code.
2400	* @param pVCpu The cross context virtual CPU structure.
2401	* @param pSvmTransient Pointer to the SVM-transient structure.
2402	*
2403	* @remarks No-long-jump zone!!!
2404	*/
2405	static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2406	{
2407	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2408
2409	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2410	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2411	Assert(pVmcb);
2412
2413	pVmcb->guest.u64RIP = pCtx->rip;
2414	pVmcb->guest.u64RSP = pCtx->rsp;
2415	pVmcb->guest.u64RFlags = pCtx->eflags.u;
2416	pVmcb->guest.u64RAX = pCtx->rax;
2417
2418	bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2419	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2420
2421	int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2422	AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2423	hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2424	hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2425	hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2426
2427	ASMSetFlags(fEFlags);
2428
2429	if (!fIsNestedGuest)
2430	{
2431	/* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2432	otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2433	hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2434	hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2435	}
2436
2437	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2438	uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2439	\| HM_CHANGED_GUEST_RFLAGS
2440	\| HM_CHANGED_GUEST_GPRS_MASK
2441	\| HM_CHANGED_GUEST_X87
2442	\| HM_CHANGED_GUEST_SSE_AVX
2443	\| HM_CHANGED_GUEST_OTHER_XSAVE
2444	\| HM_CHANGED_GUEST_XCRx
2445	\| HM_CHANGED_GUEST_TSC_AUX
2446	\| HM_CHANGED_GUEST_OTHER_MSRS;
2447	if (fIsNestedGuest)
2448	fUnusedMask \|= HM_CHANGED_SVM_XCPT_INTERCEPTS
2449	\| HM_CHANGED_GUEST_APIC_TPR;
2450
2451	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2452	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2453
2454	#ifdef VBOX_STRICT
2455	/*
2456	* All of the guest-CPU state and SVM keeper bits should be exported here by now,
2457	* except for the host-context and/or shared host-guest context bits.
2458	*/
2459	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2460	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2461	("fCtxChanged=%#RX64\n", fCtxChanged));
2462
2463	/*
2464	* If we need to log state that isn't always imported, we'll need to import them here.
2465	* See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2466	*/
2467	hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags /, 0 / uVerbose */);
2468	#endif
2469
2470	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2471	return VINF_SUCCESS;
2472	}
2473
2474	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2475
2476	/**
2477	* Merges the guest and nested-guest MSR permission bitmap.
2478	*
2479	* If the guest is intercepting an MSR we need to intercept it regardless of
2480	* whether the nested-guest is intercepting it or not.
2481	*
2482	* @param pHostCpu The HM physical-CPU structure.
2483	* @param pVCpu The cross context virtual CPU structure.
2484	*
2485	* @remarks No-long-jmp zone!!!
2486	*/
2487	DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2488	{
2489	uint64_t const pu64GstMsrpm = (uint64_t const )pVCpu->hmr0.s.svm.pvMsrBitmap;
2490	uint64_t const pu64NstGstMsrpm = (uint64_t const )&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2491	uint64_t pu64DstMsrpm = (uint64_t )pHostCpu->n.svm.pvNstGstMsrpm;
2492
2493	/* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2494	uint32_t const offRsvdQwords = 0x1800 >> 3;
2495	for (uint32_t i = 0; i < offRsvdQwords; i++)
2496	pu64DstMsrpm[i] = pu64NstGstMsrpm[i] \| pu64GstMsrpm[i];
2497	}
2498
2499
2500	/**
2501	* Caches the nested-guest VMCB fields before we modify them for execution using
2502	* hardware-assisted SVM.
2503	*
2504	* @returns true if the VMCB was previously already cached, false otherwise.
2505	* @param pVCpu The cross context virtual CPU structure.
2506	*
2507	* @sa HMNotifySvmNstGstVmexit.
2508	*/
2509	static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2510	{
2511	/*
2512	* Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2513	* Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2514	*
2515	* Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2516	* cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2517	*/
2518	PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2519	bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2520	if (!fWasCached)
2521	{
2522	PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2523	PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2524	pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2525	pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2526	pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2527	pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2528	pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2529	pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2530	pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2531	pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2532	pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2533	pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2534	pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2535	pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2536	pVmcbNstGstCache->fCacheValid = true;
2537	Log4Func(("Cached VMCB fields\n"));
2538	}
2539
2540	return fWasCached;
2541	}
2542
2543
2544	/**
2545	* Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2546	*
2547	* This is done the first time we enter nested-guest execution using SVM R0
2548	* until the nested-guest \#VMEXIT (not to be confused with physical CPU
2549	* \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2550	*
2551	* @param pVCpu The cross context virtual CPU structure.
2552	*/
2553	static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2554	{
2555	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2556	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2557
2558	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2559
2560	/*
2561	* First cache the nested-guest VMCB fields we may potentially modify.
2562	*/
2563	bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2564	if (!fVmcbCached)
2565	{
2566	/*
2567	* The IOPM of the nested-guest can be ignored because the the guest always
2568	* intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2569	* than the nested-guest IOPM and swap the field back on the #VMEXIT.
2570	*/
2571	pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2572
2573	/*
2574	* Use the same nested-paging as the outer guest. We can't dynamically switch off
2575	* nested-paging suddenly while executing a VM (see assertion at the end of
2576	* Trap0eHandler() in PGMAllBth.h).
2577	*/
2578	pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2579
2580	/* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2581	pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2582
2583	/*
2584	* Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2585	* to the nested-guest intercepts and we always run with V_INTR_MASKING.
2586	*/
2587	pVCpu->hmr0.s.svm.fSyncVTpr = false;
2588
2589	# ifdef DEBUG_ramshankar
2590	/* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2591	pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2592	# endif
2593
2594	/*
2595	* If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2596	* need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2597	*/
2598	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2599	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_VMSAVE
2600	\| SVM_CTRL_INTERCEPT_VMLOAD;
2601
2602	/*
2603	* If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2604	* CLGI/STGI instructions executed by the nested-guest.
2605	*/
2606	if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2607	pVmcbNstGstCtrl->u64InterceptCtrl \|= SVM_CTRL_INTERCEPT_CLGI
2608	\| SVM_CTRL_INTERCEPT_STGI;
2609
2610	/* Merge the guest and nested-guest intercepts. */
2611	hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2612
2613	/* Update the VMCB clean bits. */
2614	pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2615	}
2616	else
2617	{
2618	Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2619	Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2620	Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2621	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2622	}
2623	}
2624
2625	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2626
2627	/**
2628	* Exports the state shared between the host and guest (or nested-guest) into
2629	* the VMCB.
2630	*
2631	* @param pVCpu The cross context virtual CPU structure.
2632	* @param pVmcb Pointer to the VM control block.
2633	*
2634	* @remarks No-long-jump zone!!!
2635	*/
2636	static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2637	{
2638	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2639	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2640
2641	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2642	hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2643
2644	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2645	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2646	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2647	}
2648
2649
2650	/**
2651	* Worker for SVMR0ImportStateOnDemand.
2652	*
2653	* @param pVCpu The cross context virtual CPU structure.
2654	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2655	*/
2656	static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2657	{
2658	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2659
2660	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2661	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2662	PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2663	PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2664
2665	/*
2666	* We disable interrupts to make the updating of the state and in particular
2667	* the fExtrn modification atomic wrt to preemption hooks.
2668	*/
2669	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2670
2671	fWhat &= pCtx->fExtrn;
2672	if (fWhat)
2673	{
2674	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2675	if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2676	{
2677	if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2678	{
2679	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2680	Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2681	CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2682	}
2683	}
2684
2685	if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2686	{
2687	if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2688	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2689	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2690	}
2691	#endif
2692
2693	if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2694	CPUMUpdateInterruptShadowEx(pCtx, pVmcbCtrl->IntShadow.n.u1IntShadow, pVmcbGuest->u64RIP);
2695
2696	if (fWhat & CPUMCTX_EXTRN_RIP)
2697	pCtx->rip = pVmcbGuest->u64RIP;
2698
2699	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2700	{
2701	pCtx->eflags.u = pVmcbGuest->u64RFlags;
2702	if (pVCpu->hmr0.s.fClearTrapFlag)
2703	{
2704	pVCpu->hmr0.s.fClearTrapFlag = false;
2705	pCtx->eflags.Bits.u1TF = 0;
2706	}
2707	}
2708
2709	if (fWhat & CPUMCTX_EXTRN_RSP)
2710	pCtx->rsp = pVmcbGuest->u64RSP;
2711
2712	if (fWhat & CPUMCTX_EXTRN_RAX)
2713	pCtx->rax = pVmcbGuest->u64RAX;
2714
2715	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2716	{
2717	if (fWhat & CPUMCTX_EXTRN_CS)
2718	{
2719	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2720	/* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2721	/** @todo SELM might need to be fixed as it too should not care about the
2722	* granularity bit. See @bugref{6785}. */
2723	if ( !pCtx->cs.Attr.n.u1Granularity
2724	&& pCtx->cs.Attr.n.u1Present
2725	&& pCtx->cs.u32Limit > UINT32_C(0xfffff))
2726	{
2727	Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2728	pCtx->cs.Attr.n.u1Granularity = 1;
2729	}
2730	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2731	}
2732	if (fWhat & CPUMCTX_EXTRN_SS)
2733	{
2734	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2735	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2736	/*
2737	* Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2738	* VMCB and uses that and thus it's possible that when the CPL changes during
2739	* guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2740	* AMD Fusion CPUs with 64-bit guests.
2741	*
2742	* See AMD spec. 15.5.1 "Basic operation".
2743	*/
2744	Assert(!(pVmcbGuest->u8CPL & ~0x3));
2745	uint8_t const uCpl = pVmcbGuest->u8CPL;
2746	if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2747	pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2748	}
2749	if (fWhat & CPUMCTX_EXTRN_DS)
2750	{
2751	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2752	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2753	}
2754	if (fWhat & CPUMCTX_EXTRN_ES)
2755	{
2756	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2757	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2758	}
2759	if (fWhat & CPUMCTX_EXTRN_FS)
2760	{
2761	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2762	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2763	}
2764	if (fWhat & CPUMCTX_EXTRN_GS)
2765	{
2766	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2767	HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2768	}
2769	}
2770
2771	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2772	{
2773	if (fWhat & CPUMCTX_EXTRN_TR)
2774	{
2775	/*
2776	* Fixup TR attributes so it's compatible with Intel. Important when saved-states
2777	* are used between Intel and AMD, see @bugref{6208#c39}.
2778	* ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2779	*/
2780	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2781	if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2782	{
2783	if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2784	\|\| CPUMIsGuestInLongModeEx(pCtx))
2785	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2786	else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2787	pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2788	}
2789	}
2790
2791	if (fWhat & CPUMCTX_EXTRN_LDTR)
2792	HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2793
2794	if (fWhat & CPUMCTX_EXTRN_GDTR)
2795	{
2796	pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2797	pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2798	}
2799
2800	if (fWhat & CPUMCTX_EXTRN_IDTR)
2801	{
2802	pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2803	pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2804	}
2805	}
2806
2807	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2808	{
2809	pCtx->msrSTAR = pVmcbGuest->u64STAR;
2810	pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2811	pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2812	pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2813	}
2814
2815	if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2816	&& !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2817	{
2818	pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2819	pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2820	pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2821	}
2822
2823	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2824	pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2825
2826	if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2827	{
2828	if (fWhat & CPUMCTX_EXTRN_DR6)
2829	{
2830	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2831	pCtx->dr[6] = pVmcbGuest->u64DR6;
2832	else
2833	CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2834	}
2835
2836	if (fWhat & CPUMCTX_EXTRN_DR7)
2837	{
2838	if (!pVCpu->hmr0.s.fUsingHyperDR7)
2839	pCtx->dr[7] = pVmcbGuest->u64DR7;
2840	else
2841	Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2842	}
2843	}
2844
2845	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2846	{
2847	if (fWhat & CPUMCTX_EXTRN_CR0)
2848	{
2849	/* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2850	uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS \| X86_CR0_MP))
2851	\| (pVmcbGuest->u64CR0 & (X86_CR0_TS \| X86_CR0_MP));
2852	VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2853	CPUMSetGuestCR0(pVCpu, uCr0);
2854	VMMRZCallRing3Enable(pVCpu);
2855	}
2856
2857	if (fWhat & CPUMCTX_EXTRN_CR2)
2858	pCtx->cr2 = pVmcbGuest->u64CR2;
2859
2860	if (fWhat & CPUMCTX_EXTRN_CR3)
2861	{
2862	if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2863	&& pCtx->cr3 != pVmcbGuest->u64CR3)
2864	{
2865	CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2866	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2867	}
2868	}
2869
2870	/* Changes to CR4 are always intercepted. */
2871	}
2872
2873	/* Update fExtrn. */
2874	pCtx->fExtrn &= ~fWhat;
2875
2876	/* If everything has been imported, clear the HM keeper bit. */
2877	if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2878	{
2879	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2880	Assert(!pCtx->fExtrn);
2881	}
2882	}
2883	else
2884	Assert(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2885
2886	ASMSetFlags(fEFlags);
2887
2888	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2889
2890	/*
2891	* Honor any pending CR3 updates.
2892	*
2893	* Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2894	* -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2895	* -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2896	* handling -> hmR0SvmImportGuestState() and here we are.
2897	*
2898	* The reason for such complicated handling is because VM-exits that call into PGM expect
2899	* CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2900	* postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2901	* VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2902	*
2903	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2904	* and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2905	*/
2906	if ( VMMRZCallRing3IsEnabled(pVCpu)
2907	&& VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2908	{
2909	AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2910	PGMUpdateCR3(pVCpu, pCtx->cr3);
2911	}
2912	}
2913
2914
2915	/**
2916	* Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2917	* context.
2918	*
2919	* Currently there is no residual state left in the CPU that is not updated in the
2920	* VMCB.
2921	*
2922	* @returns VBox status code.
2923	* @param pVCpu The cross context virtual CPU structure.
2924	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2925	*/
2926	VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2927	{
2928	hmR0SvmImportGuestState(pVCpu, fWhat);
2929	return VINF_SUCCESS;
2930	}
2931
2932
2933	/**
2934	* Gets SVM \#VMEXIT auxiliary information.
2935	*
2936	* @returns VBox status code.
2937	* @param pVCpu The cross context virtual CPU structure.
2938	* @param pSvmExitAux Where to store the auxiliary info.
2939	*/
2940	VMMR0DECL(int) SVMR0GetExitAuxInfo(PVMCPUCC pVCpu, PSVMEXITAUX pSvmExitAux)
2941	{
2942	PCSVMTRANSIENT pSvmTransient = pVCpu->hmr0.s.svm.pSvmTransient;
2943	if (RT_LIKELY(pSvmTransient))
2944	{
2945	PCSVMVMCB pVmcb = pSvmTransient->pVmcb;
2946	if (RT_LIKELY(pVmcb))
2947	{
2948	pSvmExitAux->u64ExitCode = pVmcb->ctrl.u64ExitCode;
2949	pSvmExitAux->u64ExitInfo1 = pVmcb->ctrl.u64ExitInfo1;
2950	pSvmExitAux->u64ExitInfo2 = pVmcb->ctrl.u64ExitInfo2;
2951	pSvmExitAux->ExitIntInfo = pVmcb->ctrl.ExitIntInfo;
2952	return VINF_SUCCESS;
2953	}
2954	return VERR_SVM_IPE_5;
2955	}
2956	return VERR_NOT_AVAILABLE;
2957	}
2958
2959
2960	/**
2961	* Does the necessary state syncing before returning to ring-3 for any reason
2962	* (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2963	*
2964	* @param pVCpu The cross context virtual CPU structure.
2965	* @param fImportState Whether to import the guest state from the VMCB back
2966	* to the guest-CPU context.
2967	*
2968	* @remarks No-long-jmp zone!!!
2969	*/
2970	static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2971	{
2972	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2973	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2974
2975	/*
2976	* !!! IMPORTANT !!!
2977	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2978	*/
2979
2980	/* Save the guest state if necessary. */
2981	if (fImportState)
2982	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2983
2984	/* Restore host FPU state if necessary and resync on next R0 reentry. */
2985	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2986	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2987
2988	/*
2989	* Restore host debug registers if necessary and resync on next R0 reentry.
2990	*/
2991	#ifdef VBOX_STRICT
2992	if (CPUMIsHyperDebugStateActive(pVCpu))
2993	{
2994	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2995	Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2996	Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2997	}
2998	#endif
2999	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3000	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3001	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3002
3003	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3004	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3005	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3006	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3007	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3008	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3009	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3010
3011	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3012	}
3013
3014
3015	/**
3016	* Leaves the AMD-V session.
3017	*
3018	* Only used while returning to ring-3 either due to longjump or exits to
3019	* ring-3.
3020	*
3021	* @returns VBox status code.
3022	* @param pVCpu The cross context virtual CPU structure.
3023	*/
3024	static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3025	{
3026	HM_DISABLE_PREEMPT(pVCpu);
3027	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3028	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3029
3030	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3031	and done this from the SVMR0ThreadCtxCallback(). */
3032	if (!pVCpu->hmr0.s.fLeaveDone)
3033	{
3034	hmR0SvmLeave(pVCpu, true /* fImportState */);
3035	pVCpu->hmr0.s.fLeaveDone = true;
3036	}
3037
3038	/*
3039	* !!! IMPORTANT !!!
3040	* If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3041	*/
3042
3043	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3044	/* Deregister hook now that we've left HM context before re-enabling preemption. */
3045	VMMR0ThreadCtxHookDisable(pVCpu);
3046
3047	/* Leave HM context. This takes care of local init (term). */
3048	int rc = HMR0LeaveCpu(pVCpu);
3049
3050	HM_RESTORE_PREEMPT();
3051	return rc;
3052	}
3053
3054
3055	/**
3056	* VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3057	* any remaining host state) before we go back to ring-3 due to an assertion.
3058	*
3059	* @param pVCpu The cross context virtual CPU structure.
3060	*/
3061	VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3062	{
3063	/*
3064	* !!! IMPORTANT !!!
3065	* If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3066	* to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3067	*/
3068	VMMR0AssertionRemoveNotification(pVCpu);
3069	VMMRZCallRing3Disable(pVCpu);
3070	HM_DISABLE_PREEMPT(pVCpu);
3071
3072	/* Import the entire guest state. */
3073	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3074
3075	/* Restore host FPU state if necessary and resync on next R0 reentry. */
3076	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3077
3078	/* Restore host debug registers if necessary and resync on next R0 reentry. */
3079	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3080
3081	/* Deregister the hook now that we've left HM context before re-enabling preemption. */
3082	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3083	VMMR0ThreadCtxHookDisable(pVCpu);
3084
3085	/* Leave HM context. This takes care of local init (term). */
3086	HMR0LeaveCpu(pVCpu);
3087
3088	HM_RESTORE_PREEMPT();
3089	return VINF_SUCCESS;
3090	}
3091
3092
3093	/**
3094	* Take necessary actions before going back to ring-3.
3095	*
3096	* An action requires us to go back to ring-3. This function does the necessary
3097	* steps before we can safely return to ring-3. This is not the same as longjmps
3098	* to ring-3, this is voluntary.
3099	*
3100	* @returns Strict VBox status code.
3101	* @param pVCpu The cross context virtual CPU structure.
3102	* @param rcExit The reason for exiting to ring-3. Can be
3103	* VINF_VMM_UNKNOWN_RING3_CALL.
3104	*/
3105	static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3106	{
3107	Assert(pVCpu);
3108	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3109
3110	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3111	VMMRZCallRing3Disable(pVCpu);
3112	Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3113	pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3114
3115	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3116	if (pVCpu->hm.s.Event.fPending)
3117	{
3118	hmR0SvmPendingEventToTrpmTrap(pVCpu);
3119	Assert(!pVCpu->hm.s.Event.fPending);
3120	}
3121
3122	/* Sync. the necessary state for going back to ring-3. */
3123	hmR0SvmLeaveSession(pVCpu);
3124	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3125
3126	/* Thread-context hooks are unregistered at this point!!! */
3127	/* Ring-3 callback notifications are unregistered at this point!!! */
3128
3129	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3130	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3131	\| CPUM_CHANGED_LDTR
3132	\| CPUM_CHANGED_GDTR
3133	\| CPUM_CHANGED_IDTR
3134	\| CPUM_CHANGED_TR
3135	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
3136	if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3137	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3138	{
3139	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3140	}
3141
3142	/* Update the exit-to-ring 3 reason. */
3143	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3144
3145	/* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3146	if ( rcExit != VINF_EM_RAW_INTERRUPT
3147	\|\| CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3148	{
3149	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3150	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3151	}
3152
3153	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3154	VMMRZCallRing3Enable(pVCpu);
3155
3156	/*
3157	* If we're emulating an instruction, we shouldn't have any TRPM traps pending
3158	* and if we're injecting an event we should have a TRPM trap pending.
3159	*/
3160	AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu),
3161	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3162	VERR_SVM_IPE_5);
3163	AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu),
3164	pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3165	VERR_SVM_IPE_4);
3166
3167	return rcExit;
3168	}
3169
3170
3171	/**
3172	* Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3173	* intercepts.
3174	*
3175	* @param pVCpu The cross context virtual CPU structure.
3176	* @param pVmcb Pointer to the VM control block.
3177	*
3178	* @remarks No-long-jump zone!!!
3179	*/
3180	static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3181	{
3182	/*
3183	* Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3184	* and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3185	* RDTSC/RDTSCP as well.
3186	*/
3187	bool fParavirtTsc;
3188	uint64_t uTscOffset;
3189	bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3190
3191	bool fIntercept;
3192	if (fCanUseRealTsc)
3193	fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3194	else
3195	{
3196	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP);
3197	fIntercept = true;
3198	}
3199
3200	if (!fIntercept)
3201	{
3202	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3203	/* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3204	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3205	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3206	#endif
3207
3208	/* Update the TSC offset in the VMCB and the relevant clean bits. */
3209	pVmcb->ctrl.u64TSCOffset = uTscOffset;
3210	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3211	}
3212
3213	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3214	information before every VM-entry, hence we have nothing to do here at the moment. */
3215	if (fParavirtTsc)
3216	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3217	}
3218
3219
3220	/**
3221	* Sets an event as a pending event to be injected into the guest.
3222	*
3223	* @param pVCpu The cross context virtual CPU structure.
3224	* @param pEvent Pointer to the SVM event.
3225	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3226	* page-fault.
3227	*
3228	* @remarks Statistics counter assumes this is a guest event being reflected to
3229	* the guest i.e. 'StatInjectPendingReflect' is incremented always.
3230	*/
3231	DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3232	{
3233	Assert(!pVCpu->hm.s.Event.fPending);
3234	Assert(pEvent->n.u1Valid);
3235
3236	pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3237	pVCpu->hm.s.Event.fPending = true;
3238	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3239
3240	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3241	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3242	}
3243
3244
3245	/**
3246	* Sets an divide error (\#DE) exception as pending-for-injection into the VM.
3247	*
3248	* @param pVCpu The cross context virtual CPU structure.
3249	*/
3250	DECLINLINE(void) hmR0SvmSetPendingXcptDE(PVMCPUCC pVCpu)
3251	{
3252	SVMEVENT Event;
3253	Event.u = 0;
3254	Event.n.u1Valid = 1;
3255	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3256	Event.n.u8Vector = X86_XCPT_DE;
3257	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3258	}
3259
3260
3261	/**
3262	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3263	*
3264	* @param pVCpu The cross context virtual CPU structure.
3265	*/
3266	DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3267	{
3268	SVMEVENT Event;
3269	Event.u = 0;
3270	Event.n.u1Valid = 1;
3271	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3272	Event.n.u8Vector = X86_XCPT_UD;
3273	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3274	}
3275
3276
3277	/**
3278	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
3279	*
3280	* @param pVCpu The cross context virtual CPU structure.
3281	*/
3282	DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3283	{
3284	SVMEVENT Event;
3285	Event.u = 0;
3286	Event.n.u1Valid = 1;
3287	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3288	Event.n.u8Vector = X86_XCPT_DB;
3289	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3290	}
3291
3292
3293	/**
3294	* Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3295	*
3296	* @param pVCpu The cross context virtual CPU structure.
3297	* @param u32ErrCode The error-code for the page-fault.
3298	* @param uFaultAddress The page fault address (CR2).
3299	*
3300	* @remarks This updates the guest CR2 with @a uFaultAddress!
3301	*/
3302	DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3303	{
3304	SVMEVENT Event;
3305	Event.u = 0;
3306	Event.n.u1Valid = 1;
3307	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3308	Event.n.u8Vector = X86_XCPT_PF;
3309	Event.n.u1ErrorCodeValid = 1;
3310	Event.n.u32ErrorCode = u32ErrCode;
3311
3312	/* Update CR2 of the guest. */
3313	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3314	if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3315	{
3316	pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3317	/* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3318	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3319	}
3320
3321	hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3322	}
3323
3324
3325	/**
3326	* Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3327	*
3328	* @param pVCpu The cross context virtual CPU structure.
3329	*/
3330	DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3331	{
3332	SVMEVENT Event;
3333	Event.u = 0;
3334	Event.n.u1Valid = 1;
3335	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3336	Event.n.u8Vector = X86_XCPT_MF;
3337	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3338	}
3339
3340
3341	/**
3342	* Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3343	*
3344	* @param pVCpu The cross context virtual CPU structure.
3345	*/
3346	DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3347	{
3348	SVMEVENT Event;
3349	Event.u = 0;
3350	Event.n.u1Valid = 1;
3351	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3352	Event.n.u8Vector = X86_XCPT_DF;
3353	Event.n.u1ErrorCodeValid = 1;
3354	Event.n.u32ErrorCode = 0;
3355	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3356	}
3357
3358
3359	/**
3360	* Injects an event into the guest upon VMRUN by updating the relevant field
3361	* in the VMCB.
3362	*
3363	* @param pVCpu The cross context virtual CPU structure.
3364	* @param pVmcb Pointer to the guest VM control block.
3365	* @param pEvent Pointer to the event.
3366	*
3367	* @remarks No-long-jump zone!!!
3368	* @remarks Requires CR0!
3369	*/
3370	DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3371	{
3372	Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3373	pVmcb->ctrl.EventInject.u = pEvent->u;
3374	if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3375	\|\| pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3376	{
3377	Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3378	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3379	}
3380	else
3381	STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3382	RT_NOREF(pVCpu);
3383
3384	Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3385	(uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3386	}
3387
3388
3389
3390	/**
3391	* Converts any TRPM trap into a pending HM event. This is typically used when
3392	* entering from ring-3 (not longjmp returns).
3393	*
3394	* @param pVCpu The cross context virtual CPU structure.
3395	*/
3396	static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3397	{
3398	Assert(TRPMHasTrap(pVCpu));
3399	Assert(!pVCpu->hm.s.Event.fPending);
3400
3401	uint8_t uVector;
3402	TRPMEVENT enmTrpmEvent;
3403	uint32_t uErrCode;
3404	RTGCUINTPTR GCPtrFaultAddress;
3405	uint8_t cbInstr;
3406
3407	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3408	AssertRC(rc);
3409
3410	SVMEVENT Event;
3411	Event.u = 0;
3412	Event.n.u1Valid = 1;
3413	Event.n.u8Vector = uVector;
3414
3415	/* Refer AMD spec. 15.20 "Event Injection" for the format. */
3416	if (enmTrpmEvent == TRPM_TRAP)
3417	{
3418	Event.n.u3Type = SVM_EVENT_EXCEPTION;
3419	switch (uVector)
3420	{
3421	case X86_XCPT_NMI:
3422	{
3423	Event.n.u3Type = SVM_EVENT_NMI;
3424	break;
3425	}
3426
3427	case X86_XCPT_BP:
3428	case X86_XCPT_OF:
3429	AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3430	RT_FALL_THRU();
3431
3432	case X86_XCPT_PF:
3433	case X86_XCPT_DF:
3434	case X86_XCPT_TS:
3435	case X86_XCPT_NP:
3436	case X86_XCPT_SS:
3437	case X86_XCPT_GP:
3438	case X86_XCPT_AC:
3439	{
3440	Event.n.u1ErrorCodeValid = 1;
3441	Event.n.u32ErrorCode = uErrCode;
3442	break;
3443	}
3444	}
3445	}
3446	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3447	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3448	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3449	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3450	else
3451	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3452
3453	rc = TRPMResetTrap(pVCpu);
3454	AssertRC(rc);
3455
3456	Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3457	!!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3458
3459	hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3460	}
3461
3462
3463	/**
3464	* Converts any pending SVM event into a TRPM trap. Typically used when leaving
3465	* AMD-V to execute any instruction.
3466	*
3467	* @param pVCpu The cross context virtual CPU structure.
3468	*/
3469	static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3470	{
3471	Assert(pVCpu->hm.s.Event.fPending);
3472	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3473
3474	SVMEVENT Event;
3475	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3476
3477	uint8_t uVector = Event.n.u8Vector;
3478	TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3479
3480	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3481
3482	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3483	AssertRC(rc);
3484
3485	if (Event.n.u1ErrorCodeValid)
3486	TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3487
3488	if ( enmTrapType == TRPM_TRAP
3489	&& uVector == X86_XCPT_PF)
3490	{
3491	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3492	Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3493	}
3494	else if (enmTrapType == TRPM_SOFTWARE_INT)
3495	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3496	pVCpu->hm.s.Event.fPending = false;
3497	}
3498
3499
3500	/**
3501	* Sets the virtual interrupt intercept control in the VMCB.
3502	*
3503	* @param pVCpu The cross context virtual CPU structure.
3504	* @param pVmcb Pointer to the VM control block.
3505	*/
3506	static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3507	{
3508	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3509
3510	/*
3511	* When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3512	* is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3513	* (updating ISR at the right time) and inject the interrupt.
3514	*
3515	* With AVIC is supported, we could make use of the asynchronously delivery without
3516	* #VMEXIT and we would be passing the AVIC page to SVM.
3517	*
3518	* In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3519	* is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3520	*/
3521	Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3522	pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3523	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3524	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3525	Log4(("Set VINTR intercept\n"));
3526	}
3527
3528
3529	/**
3530	* Clears the virtual interrupt intercept control in the VMCB as
3531	* we are figured the guest is unable process any interrupts
3532	* at this point of time.
3533	*
3534	* @param pVCpu The cross context virtual CPU structure.
3535	* @param pVmcb Pointer to the VM control block.
3536	*/
3537	static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3538	{
3539	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3540
3541	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3542	if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3543	\|\| (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3544	{
3545	pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3546	pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3547	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3548	Log4(("Cleared VINTR intercept\n"));
3549	}
3550	}
3551
3552
3553	/**
3554	* Evaluates the event to be delivered to the guest and sets it as the pending
3555	* event.
3556	*
3557	* @returns Strict VBox status code.
3558	* @param pVCpu The cross context virtual CPU structure.
3559	* @param pSvmTransient Pointer to the SVM transient structure.
3560	*/
3561	static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3562	{
3563	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3564	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3565	\| CPUMCTX_EXTRN_RFLAGS
3566	\| CPUMCTX_EXTRN_INHIBIT_INT
3567	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3568
3569	Assert(!pVCpu->hm.s.Event.fPending);
3570	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3571	Assert(pVmcb);
3572
3573	bool const fGif = CPUMGetGuestGif(pCtx);
3574	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(pCtx);
3575	bool const fBlockNmi = CPUMAreInterruptsInhibitedByNmi(pCtx);
3576
3577	Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3578	fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC),
3579	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3580
3581	/** @todo SMI. SMIs take priority over NMIs. */
3582
3583	/*
3584	* Check if the guest or nested-guest can receive NMIs.
3585	* Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3586	* NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3587	*/
3588	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3589	&& !fBlockNmi)
3590	{
3591	if ( fGif
3592	&& !fIntShadow)
3593	{
3594	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3595	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3596	{
3597	Log4(("Intercepting NMI -> #VMEXIT\n"));
3598	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3599	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3600	}
3601	#endif
3602	Log4(("Setting NMI pending for injection\n"));
3603	SVMEVENT Event;
3604	Event.u = 0;
3605	Event.n.u1Valid = 1;
3606	Event.n.u8Vector = X86_XCPT_NMI;
3607	Event.n.u3Type = SVM_EVENT_NMI;
3608	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3609	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3610	}
3611	else if (!fGif)
3612	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3613	else if (!pSvmTransient->fIsNestedGuest)
3614	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3615	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3616	}
3617	/*
3618	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3619	* returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3620	* it from the APIC device.
3621	*
3622	* For nested-guests, physical interrupts always take priority over virtual interrupts.
3623	* We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3624	* do that work when we execute nested-guest code esp. since all the required information
3625	* is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3626	* the virtual interrupt controller.
3627	*
3628	* See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3629	*/
3630	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
3631	&& !pVCpu->hm.s.fSingleInstruction)
3632	{
3633	bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u & X86_EFL_IF)
3634	: CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3635	if ( fGif
3636	&& !fBlockInt
3637	&& !fIntShadow)
3638	{
3639	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3640	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3641	{
3642	Log4(("Intercepting INTR -> #VMEXIT\n"));
3643	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3644	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3645	}
3646	#endif
3647	uint8_t u8Interrupt;
3648	int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3649	if (RT_SUCCESS(rc))
3650	{
3651	Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3652	SVMEVENT Event;
3653	Event.u = 0;
3654	Event.n.u1Valid = 1;
3655	Event.n.u8Vector = u8Interrupt;
3656	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3657	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3658	}
3659	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3660	{
3661	/*
3662	* AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3663	* updated eventually when the TPR is written by the guest.
3664	*/
3665	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3666	}
3667	else
3668	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3669	}
3670	else if (!fGif)
3671	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3672	else if (!pSvmTransient->fIsNestedGuest)
3673	hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3674	/* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3675	}
3676
3677	return VINF_SUCCESS;
3678	}
3679
3680
3681	/**
3682	* Injects any pending events into the guest (or nested-guest).
3683	*
3684	* @param pVCpu The cross context virtual CPU structure.
3685	* @param pVmcb Pointer to the VM control block.
3686	*
3687	* @remarks Must only be called when we are guaranteed to enter
3688	* hardware-assisted SVM execution and not return to ring-3
3689	* prematurely.
3690	*/
3691	static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3692	{
3693	Assert(!TRPMHasTrap(pVCpu));
3694	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3695
3696	bool const fIntShadow = CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx);
3697	#ifdef VBOX_STRICT
3698	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3699	bool const fGif = CPUMGetGuestGif(pCtx);
3700	bool fAllowInt = fGif;
3701	if (fGif)
3702	{
3703	/*
3704	* For nested-guests we have no way to determine if we're injecting a physical or
3705	* virtual interrupt at this point. Hence the partial verification below.
3706	*/
3707	if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3708	fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) \|\| CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3709	else
3710	fAllowInt = RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
3711	}
3712	#endif
3713
3714	if (pVCpu->hm.s.Event.fPending)
3715	{
3716	SVMEVENT Event;
3717	Event.u = pVCpu->hm.s.Event.u64IntInfo;
3718	Assert(Event.n.u1Valid);
3719
3720	/*
3721	* Validate event injection pre-conditions.
3722	*/
3723	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3724	{
3725	Assert(fAllowInt);
3726	Assert(!fIntShadow);
3727	}
3728	else if (Event.n.u3Type == SVM_EVENT_NMI)
3729	{
3730	Assert(fGif);
3731	Assert(!fIntShadow);
3732	}
3733
3734	/*
3735	* Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3736	* do this only when we are surely going to inject the NMI as otherwise if we return
3737	* to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3738	* SVM R0.
3739	*
3740	* With VT-x, this is handled by the Guest interruptibility information VMCS field
3741	* which will set the VMCS field after actually delivering the NMI which we read on
3742	* VM-exit to determine the state.
3743	*/
3744	if ( Event.n.u3Type == SVM_EVENT_NMI
3745	&& Event.n.u8Vector == X86_XCPT_NMI)
3746	CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
3747
3748	/*
3749	* Inject it (update VMCB for injection by the hardware).
3750	*/
3751	Log4(("Injecting pending HM event\n"));
3752	hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3753	pVCpu->hm.s.Event.fPending = false;
3754
3755	if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3756	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3757	else
3758	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3759	}
3760	else
3761	Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3762
3763	/*
3764	* We could have injected an NMI through IEM and continue guest execution using
3765	* hardware-assisted SVM. In which case, we would not have any events pending (above)
3766	* but we still need to intercept IRET in order to eventually clear NMI inhibition.
3767	*/
3768	if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
3769	hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3770
3771	/*
3772	* Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3773	*
3774	* For nested-guests: We need to update it too for the scenario where IEM executes
3775	* the nested-guest but execution later continues here with an interrupt shadow active.
3776	*/
3777	pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3778	}
3779
3780
3781	/**
3782	* Reports world-switch error and dumps some useful debug info.
3783	*
3784	* @param pVCpu The cross context virtual CPU structure.
3785	* @param rcVMRun The return code from VMRUN (or
3786	* VERR_SVM_INVALID_GUEST_STATE for invalid
3787	* guest-state).
3788	*/
3789	static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3790	{
3791	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3792	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3793	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3794
3795	if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3796	{
3797	#ifdef VBOX_STRICT
3798	hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3799	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3800	Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3801	Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3802	Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3803	Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3804	Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3805	Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3806	Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3807	Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3808	Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3809	Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3810
3811	Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3812	Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3813	Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3814
3815	Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3816	Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3817	Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3818	Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3819	Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3820	Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3821	Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3822	Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3823	Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3824	Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3825	Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3826	Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3827
3828	Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3829	Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3830	Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3831	Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3832	Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3833	Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3834	Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3835	Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3836	Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3837	Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3838	Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3839	Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3840	Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3841	Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3842	Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3843	Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3844	Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3845	Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3846	Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3847	Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3848
3849	Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3850
3851	Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3852	Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3853
3854	Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3855	Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3856	Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3857	Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3858	Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3859	Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3860	Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3861	Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3862	Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3863	Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3864	Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3865	Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3866	Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3867	Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3868	Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3869	Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3870	Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3871	Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3872	Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3873	Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3874
3875	Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3876	Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3877
3878	Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3879	Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3880	Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3881	Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3882
3883	Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3884	Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3885
3886	Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3887	Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3888	Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3889	Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3890
3891	Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3892	Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3893	Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3894	Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3895	Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3896	Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3897	Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3898
3899	Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3900	Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3901	Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3902	Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3903
3904	Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3905	Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3906	Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3907
3908	Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3909	Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3910	Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3911	Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3912	Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3913	Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3914	Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3915	Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3916	Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3917	Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3918	Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3919	Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3920
3921	NOREF(pVmcb);
3922	#endif /* VBOX_STRICT */
3923	}
3924	else
3925	Log4Func(("rcVMRun=%d\n", rcVMRun));
3926	}
3927
3928
3929	/**
3930	* Check per-VM and per-VCPU force flag actions that require us to go back to
3931	* ring-3 for one reason or another.
3932	*
3933	* @returns Strict VBox status code (information status code included).
3934	* @retval VINF_SUCCESS if we don't have any actions that require going back to
3935	* ring-3.
3936	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3937	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3938	* interrupts)
3939	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3940	* all EMTs to be in ring-3.
3941	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3942	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3943	* to the EM loop.
3944	*
3945	* @param pVCpu The cross context virtual CPU structure.
3946	*/
3947	static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3948	{
3949	Assert(VMMRZCallRing3IsEnabled(pVCpu));
3950
3951	/* Could happen as a result of longjump. */
3952	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3953	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3954
3955	/* Update pending interrupts into the APIC's IRR. */
3956	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3957	APICUpdatePendingInterrupts(pVCpu);
3958
3959	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3960	if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3961	? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3962	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3963	? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3964	{
3965	/* Pending PGM C3 sync. */
3966	if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3967	{
3968	int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3969	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3970	if (rc != VINF_SUCCESS)
3971	{
3972	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3973	return rc;
3974	}
3975	}
3976
3977	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3978	/* -XXX- what was that about single stepping? */
3979	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
3980	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3981	{
3982	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3983	int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
3984	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3985	return rc;
3986	}
3987
3988	/* Pending VM request packets, such as hardware interrupts. */
3989	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
3990	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
3991	{
3992	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
3993	Log4Func(("Pending VM request forcing us back to ring-3\n"));
3994	return VINF_EM_PENDING_REQUEST;
3995	}
3996
3997	/* Pending PGM pool flushes. */
3998	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3999	{
4000	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4001	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4002	return VINF_PGM_POOL_FLUSH_PENDING;
4003	}
4004
4005	/* Pending DMA requests. */
4006	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4007	{
4008	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4009	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4010	return VINF_EM_RAW_TO_R3;
4011	}
4012	}
4013
4014	return VINF_SUCCESS;
4015	}
4016
4017
4018	/**
4019	* Does the preparations before executing guest code in AMD-V.
4020	*
4021	* This may cause longjmps to ring-3 and may even result in rescheduling to the
4022	* recompiler. We must be cautious what we do here regarding committing
4023	* guest-state information into the VMCB assuming we assuredly execute the guest
4024	* in AMD-V. If we fall back to the recompiler after updating the VMCB and
4025	* clearing the common-state (TRPM/forceflags), we must undo those changes so
4026	* that the recompiler can (and should) use them when it resumes guest
4027	* execution. Otherwise such operations must be done when we can no longer
4028	* exit to ring-3.
4029	*
4030	* @returns Strict VBox status code (informational status codes included).
4031	* @retval VINF_SUCCESS if we can proceed with running the guest.
4032	* @retval VINF_* scheduling changes, we have to go back to ring-3.
4033	*
4034	* @param pVCpu The cross context virtual CPU structure.
4035	* @param pSvmTransient Pointer to the SVM transient structure.
4036	*/
4037	static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4038	{
4039	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4040
4041	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4042	if (pSvmTransient->fIsNestedGuest)
4043	{
4044	Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4045	return VINF_EM_RESCHEDULE_REM;
4046	}
4047	#endif
4048
4049	/* Check force flag actions that might require us to go back to ring-3. */
4050	VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4051	if (rc != VINF_SUCCESS)
4052	return rc;
4053
4054	if (TRPMHasTrap(pVCpu))
4055	hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4056	else if (!pVCpu->hm.s.Event.fPending)
4057	{
4058	rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4059	if ( rc != VINF_SUCCESS
4060	\|\| pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4061	{
4062	/* If a nested-guest VM-exit occurred, bail. */
4063	if (pSvmTransient->fIsNestedGuest)
4064	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4065	return rc;
4066	}
4067	}
4068
4069	/*
4070	* On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4071	* Just do it in software, see @bugref{8411}.
4072	* NB: If we could continue a task switch exit we wouldn't need to do this.
4073	*/
4074	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4075	if (RT_UNLIKELY( !g_fHmSvmFeatures
4076	&& pVCpu->hm.s.Event.fPending
4077	&& SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4078	return VINF_EM_RAW_INJECT_TRPM_EVENT;
4079
4080	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4081	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4082	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4083	#endif
4084
4085	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4086	/*
4087	* Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4088	*/
4089	if (pSvmTransient->fIsNestedGuest)
4090	hmR0SvmSetupVmcbNested(pVCpu);
4091	#endif
4092
4093	/*
4094	* Export the guest state bits that are not shared with the host in any way as we can
4095	* longjmp or get preempted in the midst of exporting some of the state.
4096	*/
4097	rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4098	AssertRCReturn(rc, rc);
4099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4100
4101	/* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4102	Assert(!pSvmTransient->fIsNestedGuest \|\| pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4103
4104	/*
4105	* If we're not intercepting TPR changes in the guest, save the guest TPR before the
4106	* world-switch so we can update it on the way back if the guest changed the TPR.
4107	*/
4108	if (pVCpu->hmr0.s.svm.fSyncVTpr)
4109	{
4110	Assert(!pSvmTransient->fIsNestedGuest);
4111	PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4112	if (pVM->hm.s.fTprPatchingActive)
4113	pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4114	else
4115	pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4116	}
4117
4118	/*
4119	* No longjmps to ring-3 from this point on!!!
4120	*
4121	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4122	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4123	*/
4124	VMMRZCallRing3Disable(pVCpu);
4125
4126	/*
4127	* We disable interrupts so that we don't miss any interrupts that would flag preemption
4128	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4129	* preemption disabled for a while. Since this is purly to aid the
4130	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4131	* disable interrupt on NT.
4132	*
4133	* We need to check for force-flags that could've possible been altered since we last
4134	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4135	* see @bugref{6398}).
4136	*
4137	* We also check a couple of other force-flags as a last opportunity to get the EMT back
4138	* to ring-3 before executing guest code.
4139	*/
4140	pSvmTransient->fEFlags = ASMIntDisableFlags();
4141	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
4142	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4143	{
4144	ASMSetFlags(pSvmTransient->fEFlags);
4145	VMMRZCallRing3Enable(pVCpu);
4146	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4147	return VINF_EM_RAW_TO_R3;
4148	}
4149	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4150	{
4151	ASMSetFlags(pSvmTransient->fEFlags);
4152	VMMRZCallRing3Enable(pVCpu);
4153	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4154	return VINF_EM_RAW_INTERRUPT;
4155	}
4156
4157	return VINF_SUCCESS;
4158	}
4159
4160
4161	/**
4162	* Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4163	* doing so.
4164	*
4165	* This means there is no backing out to ring-3 or anywhere else at this point.
4166	*
4167	* @param pVCpu The cross context virtual CPU structure.
4168	* @param pSvmTransient Pointer to the SVM transient structure.
4169	*
4170	* @remarks Called with preemption disabled.
4171	* @remarks No-long-jump zone!!!
4172	*/
4173	static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4174	{
4175	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4176	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4177
4178	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4179	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4180
4181	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4182	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4183
4184	hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4185
4186	if (!CPUMIsGuestFPUStateActive(pVCpu))
4187	{
4188	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4189	CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4190	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4191	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4192	}
4193
4194	/* Load the state shared between host and guest (FPU, debug). */
4195	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4196	hmR0SvmExportSharedState(pVCpu, pVmcb);
4197
4198	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4199	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4200
4201	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4202	RTCPUID const idHostCpu = pHostCpu->idCpu;
4203	bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4204
4205	/* Setup TSC offsetting. */
4206	if ( pSvmTransient->fUpdateTscOffsetting
4207	\|\| fMigratedHostCpu)
4208	{
4209	hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4210	pSvmTransient->fUpdateTscOffsetting = false;
4211	}
4212
4213	/* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4214	if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
4215	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4216	else
4217	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4218
4219	/* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4220	if (fMigratedHostCpu)
4221	pVmcb->ctrl.u32VmcbCleanBits = 0;
4222
4223	/* Store status of the shared guest-host state at the time of VMRUN. */
4224	pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4225	pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4226
4227	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4228	uint8_t *pbMsrBitmap;
4229	if (!pSvmTransient->fIsNestedGuest)
4230	pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4231	else
4232	{
4233	/** @todo We could perhaps optimize this by monitoring if the guest modifies its
4234	* MSRPM and only perform this if it changed also use EVEX.POR when it
4235	* does. */
4236	hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4237
4238	/* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4239	pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4240	pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4241	}
4242	#else
4243	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.svm.pvMsrBitmap;
4244	#endif
4245
4246	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4247	/* Flush the appropriate tagged-TLB entries. */
4248	hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4249	Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4250
4251	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4252
4253	TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4254	to start executing. */
4255
4256	/*
4257	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4258	* (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4259	*
4260	* This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4261	*/
4262	if ( g_CpumHostFeatures.s.fRdTscP
4263	&& !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4264	{
4265	uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4266	pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4267	if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4268	ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4269	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4270	pSvmTransient->fRestoreTscAuxMsr = true;
4271	}
4272	else
4273	{
4274	hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4275	pSvmTransient->fRestoreTscAuxMsr = false;
4276	}
4277	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4278
4279	/*
4280	* If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4281	* virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4282	* from the VMCB.
4283	*/
4284	bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4285	if (!fSupportsVmcbCleanBits)
4286	pVmcb->ctrl.u32VmcbCleanBits = 0;
4287	}
4288
4289
4290	/**
4291	* Wrapper for running the guest (or nested-guest) code in AMD-V.
4292	*
4293	* @returns VBox strict status code.
4294	* @param pVCpu The cross context virtual CPU structure.
4295	* @param HCPhysVmcb The host physical address of the VMCB.
4296	*
4297	* @remarks No-long-jump zone!!!
4298	*/
4299	DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4300	{
4301	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4302	pVCpu->cpum.GstCtx.fExtrn \|= HMSVM_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
4303	return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4304	}
4305
4306
4307	/**
4308	* Performs some essential restoration of state after running guest (or
4309	* nested-guest) code in AMD-V.
4310	*
4311	* @param pVCpu The cross context virtual CPU structure.
4312	* @param pSvmTransient Pointer to the SVM transient structure.
4313	* @param rcVMRun Return code of VMRUN.
4314	*
4315	* @remarks Called with interrupts disabled.
4316	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
4317	* unconditionally when it is safe to do so.
4318	*/
4319	static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4320	{
4321	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4322
4323	ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4324	ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4325
4326	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4327	PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4328
4329	/* TSC read must be done early for maximum accuracy. */
4330	if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4331	{
4332	if (!pSvmTransient->fIsNestedGuest)
4333	TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4334	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4335	else
4336	{
4337	/* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4338	uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4339	TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4340	}
4341	#endif
4342	}
4343
4344	if (pSvmTransient->fRestoreTscAuxMsr)
4345	{
4346	uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4347	CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4348	if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4349	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4350	}
4351
4352	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4353	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4354	TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4355	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4356
4357	Assert(!(ASMGetFlags() & X86_EFL_IF));
4358	ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4359	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4360
4361	/* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4362	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4363	{
4364	Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4365	return;
4366	}
4367
4368	pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4369	pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4370	pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4371	pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4372
4373	#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4374	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4375	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4376	#else
4377	/*
4378	* Always import the following:
4379	*
4380	* - RIP for exit optimizations and evaluating event injection on re-entry.
4381	* - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4382	* state on preemption.
4383	* - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4384	* - CS for exit optimizations.
4385	* - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4386	* assembly switcher code.
4387	* - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4388	*/
4389	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4390	\| CPUMCTX_EXTRN_RFLAGS
4391	\| CPUMCTX_EXTRN_RAX
4392	\| CPUMCTX_EXTRN_RSP
4393	\| CPUMCTX_EXTRN_CS
4394	\| CPUMCTX_EXTRN_HWVIRT
4395	\| CPUMCTX_EXTRN_INHIBIT_INT
4396	\| CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4397	\| HMSVM_CPUMCTX_SHARED_STATE);
4398	#endif
4399
4400	if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4401	&& pVCpu->hmr0.s.svm.fSyncVTpr)
4402	{
4403	Assert(!pSvmTransient->fIsNestedGuest);
4404	/* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4405	if ( pVM->hm.s.fTprPatchingActive
4406	&& (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4407	{
4408	int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4409	AssertRC(rc);
4410	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4411	}
4412	/* Sync TPR when we aren't intercepting CR8 writes. */
4413	else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4414	{
4415	int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4416	AssertRC(rc);
4417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4418	}
4419	}
4420
4421	#ifdef DEBUG_ramshankar
4422	if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4423	{
4424	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4425	hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4426	0 /* uVerbose */);
4427	}
4428	#endif
4429
4430	HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
4431	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4432	pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4433	}
4434
4435
4436	/**
4437	* Runs the guest code using AMD-V.
4438	*
4439	* @returns Strict VBox status code.
4440	* @param pVCpu The cross context virtual CPU structure.
4441	* @param pcLoops Pointer to the number of executed loops.
4442	*/
4443	static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4444	{
4445	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4446	Assert(pcLoops);
4447	Assert(*pcLoops <= cMaxResumeLoops);
4448
4449	SVMTRANSIENT SvmTransient;
4450	RT_ZERO(SvmTransient);
4451	SvmTransient.fUpdateTscOffsetting = true;
4452	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4453
4454	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4455	for (;;)
4456	{
4457	Assert(!HMR0SuspendPending());
4458	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4459
4460	/* Preparatory work for running nested-guest code, this may force us to return to
4461	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4462	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4463	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4464	if (rc != VINF_SUCCESS)
4465	break;
4466
4467	/*
4468	* No longjmps to ring-3 from this point on!!!
4469	*
4470	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4471	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4472	*/
4473	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4474	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4475
4476	/* Restore any residual host-state and save any bits shared between host and guest
4477	into the guest-CPU state. Re-enables interrupts! */
4478	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4479
4480	if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4481	\|\| SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4482	{
4483	if (rc == VINF_SUCCESS)
4484	rc = VERR_SVM_INVALID_GUEST_STATE;
4485	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4486	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4487	break;
4488	}
4489
4490	/* Handle the #VMEXIT. */
4491	HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4492	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4493	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4494	rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4495	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4496	if (rc != VINF_SUCCESS)
4497	break;
4498	if (++(*pcLoops) >= cMaxResumeLoops)
4499	{
4500	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4501	rc = VINF_EM_RAW_INTERRUPT;
4502	break;
4503	}
4504	}
4505
4506	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4507	return rc;
4508	}
4509
4510
4511	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4512	/**
4513	* Runs the nested-guest code using AMD-V.
4514	*
4515	* @returns Strict VBox status code.
4516	* @param pVCpu The cross context virtual CPU structure.
4517	* @param pcLoops Pointer to the number of executed loops. If we're switching
4518	* from the guest-code execution loop to this nested-guest
4519	* execution loop pass the remainder value, else pass 0.
4520	*/
4521	static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4522	{
4523	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4524	HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4525	Assert(pcLoops);
4526	Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4527	/** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4528	RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4529
4530	SVMTRANSIENT SvmTransient;
4531	RT_ZERO(SvmTransient);
4532	SvmTransient.fUpdateTscOffsetting = true;
4533	SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4534	SvmTransient.fIsNestedGuest = true;
4535
4536	/* Setup pointer so PGM/IEM can query #VMEXIT auxiliary info. on demand in ring-0. */
4537	pVCpu->hmr0.s.svm.pSvmTransient = &SvmTransient;
4538
4539	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4540	for (;;)
4541	{
4542	Assert(!HMR0SuspendPending());
4543	HMSVM_ASSERT_CPU_SAFE(pVCpu);
4544
4545	/* Preparatory work for running nested-guest code, this may force us to return to
4546	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4547	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4548	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4549	if ( rc != VINF_SUCCESS
4550	\|\| !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4551	break;
4552
4553	/*
4554	* No longjmps to ring-3 from this point on!!!
4555	*
4556	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4557	* better than a kernel panic. This also disables flushing of the R0-logger instance.
4558	*/
4559	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4560
4561	rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4562
4563	/* Restore any residual host-state and save any bits shared between host and guest
4564	into the guest-CPU state. Re-enables interrupts! */
4565	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4566
4567	if (RT_LIKELY( rc == VINF_SUCCESS
4568	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4569	{ /* extremely likely */ }
4570	else
4571	{
4572	/* VMRUN failed, shouldn't really happen, Guru. */
4573	if (rc != VINF_SUCCESS)
4574	break;
4575
4576	/* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4577	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4578	AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4579	rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4580	break;
4581	}
4582
4583	/* Handle the #VMEXIT. */
4584	HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4585	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4586	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4587	rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4588	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4589	if (rc == VINF_SUCCESS)
4590	{
4591	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4592	{
4593	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4594	rc = VINF_SVM_VMEXIT;
4595	}
4596	else
4597	{
4598	if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4599	continue;
4600	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4601	rc = VINF_EM_RAW_INTERRUPT;
4602	}
4603	}
4604	else
4605	Assert(rc != VINF_SVM_VMEXIT);
4606	break;
4607	/** @todo NSTSVM: handle single-stepping. */
4608	}
4609
4610	/* Ensure #VMEXIT auxiliary info. is no longer available. */
4611	pVCpu->hmr0.s.svm.pSvmTransient = NULL;
4612
4613	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4614	return rc;
4615	}
4616	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4617
4618
4619	/**
4620	* Checks if any expensive dtrace probes are enabled and we should go to the
4621	* debug loop.
4622	*
4623	* @returns true if we should use debug loop, false if not.
4624	*/
4625	static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4626	{
4627	/* It's probably faster to OR the raw 32-bit counter variables together.
4628	Since the variables are in an array and the probes are next to one
4629	another (more or less), we have good locality. So, better read
4630	eight-nine cache lines ever time and only have one conditional, than
4631	128+ conditionals, right? */
4632	return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4633	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
4634	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
4635	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
4636	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
4637	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
4638	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
4639	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
4640	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
4641	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
4642	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
4643	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
4644	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
4645	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
4646	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
4647	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
4648	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
4649	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
4650	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
4651	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4652	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4653	) != 0
4654	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4655	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4656	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4657	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4658	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
4659	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4660	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4661	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4662	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4663	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4664	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4665	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4666	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4667	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4668	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4669	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4670	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4671	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4672	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4673	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4674	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4675	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4676	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4677	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4678	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
4679	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
4680	//\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4681	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
4682	//\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4683	//\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4684	//\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4685	//\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4686	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4687	\| VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4688	\| VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4689	\| VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4690	\| VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4691	\| VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4692	) != 0
4693	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4694	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
4695	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4696	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4697	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4698	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
4699	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4700	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4701	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4702	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4703	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4704	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4705	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4706	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4707	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4708	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4709	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4710	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4711	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4712	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4713	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4714	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4715	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4716	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4717	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4718	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
4719	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
4720	//\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4721	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
4722	//\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4723	//\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4724	//\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4725	//\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4726	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4727	\| VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4728	\| VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4729	\| VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4730	\| VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4731	\| VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4732	) != 0;
4733	}
4734
4735
4736	/**
4737	* Runs the guest code using AMD-V.
4738	*
4739	* @returns Strict VBox status code.
4740	* @param pVCpu The cross context virtual CPU structure.
4741	*/
4742	VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4743	{
4744	AssertPtr(pVCpu);
4745	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4746	Assert(VMMRZCallRing3IsEnabled(pVCpu));
4747	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4748	HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4749
4750	uint32_t cLoops = 0;
4751	VBOXSTRICTRC rc;
4752	for (;;)
4753	{
4754	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4755	bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4756	#else
4757	NOREF(pCtx);
4758	bool const fInNestedGuestMode = false;
4759	#endif
4760	if (!fInNestedGuestMode)
4761	{
4762	if ( !pVCpu->hm.s.fUseDebugLoop
4763	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0SvmAnyExpensiveProbesEnabled())
4764	&& !DBGFIsStepping(pVCpu)
4765	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4766	rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4767	else
4768	rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4769	}
4770	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4771	else
4772	rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4773
4774	if (rc == VINF_SVM_VMRUN)
4775	{
4776	Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4777	continue;
4778	}
4779	if (rc == VINF_SVM_VMEXIT)
4780	{
4781	Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4782	continue;
4783	}
4784	#endif
4785	break;
4786	}
4787
4788	/* Fixup error codes. */
4789	if (rc == VERR_EM_INTERPRETER)
4790	rc = VINF_EM_RAW_EMULATE_INSTR;
4791	else if (rc == VINF_EM_RESET)
4792	rc = VINF_EM_TRIPLE_FAULT;
4793
4794	/* Prepare to return to ring-3. This will remove longjmp notifications. */
4795	rc = hmR0SvmExitToRing3(pVCpu, rc);
4796	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4797	Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4798	return rc;
4799	}
4800
4801	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4802
4803	/**
4804	* Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4805	*
4806	* @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4807	* @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4808	*/
4809	static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4810	{
4811	const uint16_t u16Port = pIoExitInfo->n.u16Port;
4812	const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4813	const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4814	const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4815	const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4816	const bool fRep = pIoExitInfo->n.u1Rep;
4817	const bool fStrIo = pIoExitInfo->n.u1Str;
4818
4819	return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4820	NULL /* pIoExitInfo */);
4821	}
4822
4823
4824	/**
4825	* Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4826	* SVM_EXIT_INVALID).
4827	*
4828	* @returns VBox status code (informational status codes included).
4829	* @param pVCpu The cross context virtual CPU structure.
4830	* @param pSvmTransient Pointer to the SVM transient structure.
4831	*/
4832	static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4833	{
4834	HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4835	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4836	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4837
4838	/*
4839	* We import the complete state here because we use separate VMCBs for the guest and the
4840	* nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4841	* the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4842	*/
4843	#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4844	do { \
4845	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4846	return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4847	} while (0)
4848
4849	/*
4850	* For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4851	* nested-guest. If it isn't, it should be handled by the (outer) guest.
4852	*/
4853	PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4854	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4855	PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4856	uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4857	uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4858	uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4859
4860	Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4861	switch (uExitCode)
4862	{
4863	case SVM_EXIT_CPUID:
4864	{
4865	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4866	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4867	return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4868	}
4869
4870	case SVM_EXIT_RDTSC:
4871	{
4872	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4873	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4874	return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4875	}
4876
4877	case SVM_EXIT_RDTSCP:
4878	{
4879	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4880	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4881	return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4882	}
4883
4884	case SVM_EXIT_MONITOR:
4885	{
4886	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4887	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4888	return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4889	}
4890
4891	case SVM_EXIT_MWAIT:
4892	{
4893	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4894	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4895	return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4896	}
4897
4898	case SVM_EXIT_HLT:
4899	{
4900	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4901	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4902	return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4903	}
4904
4905	case SVM_EXIT_MSR:
4906	{
4907	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4908	{
4909	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4910	uint16_t offMsrpm;
4911	uint8_t uMsrpmBit;
4912	int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4913	if (RT_SUCCESS(rc))
4914	{
4915	Assert(uMsrpmBit == 0 \|\| uMsrpmBit == 2 \|\| uMsrpmBit == 4 \|\| uMsrpmBit == 6);
4916	Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4917
4918	uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4919	bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4920	bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4921
4922	if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4923	\|\| (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4924	{
4925	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4926	}
4927	}
4928	else
4929	{
4930	/*
4931	* MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4932	* See AMD-V spec. "15.11 MSR Intercepts".
4933	*/
4934	Assert(rc == VERR_OUT_OF_RANGE);
4935	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4936	}
4937	}
4938	return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4939	}
4940
4941	case SVM_EXIT_IOIO:
4942	{
4943	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4944	{
4945	SVMIOIOEXITINFO IoExitInfo;
4946	IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4947	bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4948	if (fIntercept)
4949	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4950	}
4951	return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4952	}
4953
4954	case SVM_EXIT_XCPT_PF:
4955	{
4956	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4957	if (pVM->hmr0.s.fNestedPaging)
4958	{
4959	uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4960	uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4961
4962	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4963	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4964	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4965
4966	/* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4967	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4968	hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4969	return VINF_SUCCESS;
4970	}
4971	return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4972	}
4973
4974	case SVM_EXIT_XCPT_UD:
4975	{
4976	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4977	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4978	hmR0SvmSetPendingXcptUD(pVCpu);
4979	return VINF_SUCCESS;
4980	}
4981
4982	case SVM_EXIT_XCPT_MF:
4983	{
4984	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4985	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4986	return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
4987	}
4988
4989	case SVM_EXIT_XCPT_DB:
4990	{
4991	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4992	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4993	return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
4994	}
4995
4996	case SVM_EXIT_XCPT_AC:
4997	{
4998	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
4999	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5000	return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5001	}
5002
5003	case SVM_EXIT_XCPT_BP:
5004	{
5005	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5006	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5007	return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5008	}
5009
5010	case SVM_EXIT_READ_CR0:
5011	case SVM_EXIT_READ_CR3:
5012	case SVM_EXIT_READ_CR4:
5013	{
5014	uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5015	if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5016	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5017	return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5018	}
5019
5020	case SVM_EXIT_CR0_SEL_WRITE:
5021	{
5022	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5023	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5024	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5025	}
5026
5027	case SVM_EXIT_WRITE_CR0:
5028	case SVM_EXIT_WRITE_CR3:
5029	case SVM_EXIT_WRITE_CR4:
5030	case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5031	{
5032	uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5033	Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5034
5035	if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5036	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5037	return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5038	}
5039
5040	case SVM_EXIT_PAUSE:
5041	{
5042	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5043	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5044	return hmR0SvmExitPause(pVCpu, pSvmTransient);
5045	}
5046
5047	case SVM_EXIT_VINTR:
5048	{
5049	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5050	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5051	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5052	}
5053
5054	case SVM_EXIT_INTR:
5055	case SVM_EXIT_NMI:
5056	case SVM_EXIT_SMI:
5057	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5058	{
5059	/*
5060	* We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5061	*
5062	* Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5063	* get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5064	* nested-guest #VMEXIT.
5065	*
5066	* We shall import the complete state here as we may cause #VMEXITs from ring-3
5067	* while trying to inject interrupts, see comment at the top of this function.
5068	*/
5069	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5070	return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5071	}
5072
5073	case SVM_EXIT_FERR_FREEZE:
5074	{
5075	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5076	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5077	return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5078	}
5079
5080	case SVM_EXIT_INVLPG:
5081	{
5082	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5083	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5084	return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5085	}
5086
5087	case SVM_EXIT_WBINVD:
5088	{
5089	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5090	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5091	return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5092	}
5093
5094	case SVM_EXIT_INVD:
5095	{
5096	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5097	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5098	return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5099	}
5100
5101	case SVM_EXIT_RDPMC:
5102	{
5103	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5104	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5105	return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5106	}
5107
5108	default:
5109	{
5110	switch (uExitCode)
5111	{
5112	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5113	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5114	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5115	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5116	{
5117	uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5118	if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5119	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5120	return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5121	}
5122
5123	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5124	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5125	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5126	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5127	{
5128	uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5129	if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5130	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5131	return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5132	}
5133
5134	case SVM_EXIT_XCPT_DE:
5135	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5136	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5137	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5138	case SVM_EXIT_XCPT_OF:
5139	case SVM_EXIT_XCPT_BR:
5140	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5141	case SVM_EXIT_XCPT_NM:
5142	case SVM_EXIT_XCPT_DF:
5143	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5144	case SVM_EXIT_XCPT_TS:
5145	case SVM_EXIT_XCPT_NP:
5146	case SVM_EXIT_XCPT_SS:
5147	case SVM_EXIT_XCPT_GP:
5148	/* SVM_EXIT_XCPT_PF: / / Handled above. */
5149	case SVM_EXIT_XCPT_15: /* Reserved. */
5150	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5151	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5152	case SVM_EXIT_XCPT_MC:
5153	case SVM_EXIT_XCPT_XF:
5154	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5155	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5156	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5157	{
5158	uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5159	if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5160	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5161	return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5162	}
5163
5164	case SVM_EXIT_XSETBV:
5165	{
5166	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5167	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5168	return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5169	}
5170
5171	case SVM_EXIT_TASK_SWITCH:
5172	{
5173	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5174	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5175	return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5176	}
5177
5178	case SVM_EXIT_IRET:
5179	{
5180	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5181	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5182	return hmR0SvmExitIret(pVCpu, pSvmTransient);
5183	}
5184
5185	case SVM_EXIT_SHUTDOWN:
5186	{
5187	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5188	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5189	return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5190	}
5191
5192	case SVM_EXIT_VMMCALL:
5193	{
5194	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5195	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5196	return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5197	}
5198
5199	case SVM_EXIT_CLGI:
5200	{
5201	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5202	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5203	return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5204	}
5205
5206	case SVM_EXIT_STGI:
5207	{
5208	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5209	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5210	return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5211	}
5212
5213	case SVM_EXIT_VMLOAD:
5214	{
5215	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5216	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5217	return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5218	}
5219
5220	case SVM_EXIT_VMSAVE:
5221	{
5222	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5223	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5224	return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5225	}
5226
5227	case SVM_EXIT_INVLPGA:
5228	{
5229	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5230	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5231	return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5232	}
5233
5234	case SVM_EXIT_VMRUN:
5235	{
5236	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5237	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5238	return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5239	}
5240
5241	case SVM_EXIT_RSM:
5242	{
5243	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5244	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5245	hmR0SvmSetPendingXcptUD(pVCpu);
5246	return VINF_SUCCESS;
5247	}
5248
5249	case SVM_EXIT_SKINIT:
5250	{
5251	if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5252	NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5253	hmR0SvmSetPendingXcptUD(pVCpu);
5254	return VINF_SUCCESS;
5255	}
5256
5257	case SVM_EXIT_NPF:
5258	{
5259	Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5260	return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5261	}
5262
5263	case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5264	return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5265
5266	default:
5267	{
5268	AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5269	pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5270	return VERR_SVM_UNKNOWN_EXIT;
5271	}
5272	}
5273	}
5274	}
5275	/* not reached */
5276
5277	# undef NST_GST_VMEXIT_CALL_RET
5278	}
5279
5280	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5281
5282	/** @def VMEXIT_CALL_RET
5283	* Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5284	*/
5285	#ifdef DEBUG_ramshankar
5286	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5287	do { \
5288	if ((a_fDbg) == 1) \
5289	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5290	int rc = a_CallExpr; \
5291	if ((a_fDbg) == 1) \
5292	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5293	return rc; \
5294	} while (0)
5295	#else
5296	# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5297	#endif
5298
5299	/**
5300	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5301	*
5302	* @returns Strict VBox status code (informational status codes included).
5303	* @param pVCpu The cross context virtual CPU structure.
5304	* @param pSvmTransient Pointer to the SVM transient structure.
5305	*/
5306	static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5307	{
5308	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5309	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5310
5311	/*
5312	* The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5313	* for most guests under normal workloads (for some definition of "normal").
5314	*/
5315	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5316	switch (uExitCode)
5317	{
5318	case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5319	case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5320	case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5321	case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5322	case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5323	case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5324	case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5325	case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5326	case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5327	case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5328
5329	case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5330	case SVM_EXIT_INTR:
5331	case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5332
5333	case SVM_EXIT_READ_CR0:
5334	case SVM_EXIT_READ_CR3:
5335	case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5336
5337	case SVM_EXIT_CR0_SEL_WRITE:
5338	case SVM_EXIT_WRITE_CR0:
5339	case SVM_EXIT_WRITE_CR3:
5340	case SVM_EXIT_WRITE_CR4:
5341	case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5342
5343	case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5344	case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5345	case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5346	case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5347	case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5348	case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5349	case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5350	case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5351	case SVM_EXIT_XCPT_DE: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDE(pVCpu, pSvmTransient));
5352	case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5353	case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5354	case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5355	case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5356	case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5357	case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5358	case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5359	case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5360
5361	default:
5362	{
5363	switch (pSvmTransient->u64ExitCode)
5364	{
5365	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5366	case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5367	case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5368	case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5369	VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5370
5371	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5372	case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5373	case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5374	case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5375	VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5376
5377	case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5378	case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5379
5380	case SVM_EXIT_SMI:
5381	case SVM_EXIT_INIT:
5382	{
5383	/*
5384	* We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5385	* If it ever does, we want to know about it so log the exit code and bail.
5386	*/
5387	VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5388	}
5389
5390	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5391	case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5392	case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5393	case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5394	case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5395	case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5396	case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5397	#else
5398	case SVM_EXIT_CLGI:
5399	case SVM_EXIT_STGI:
5400	case SVM_EXIT_VMLOAD:
5401	case SVM_EXIT_VMSAVE:
5402	case SVM_EXIT_INVLPGA:
5403	case SVM_EXIT_VMRUN:
5404	#endif
5405	case SVM_EXIT_RSM:
5406	case SVM_EXIT_SKINIT:
5407	{
5408	hmR0SvmSetPendingXcptUD(pVCpu);
5409	return VINF_SUCCESS;
5410	}
5411
5412	/*
5413	* The remaining should only be possible when debugging or dtracing.
5414	*/
5415	case SVM_EXIT_XCPT_DE:
5416	/* SVM_EXIT_XCPT_DB: / / Handled above. */
5417	/* SVM_EXIT_XCPT_NMI: / / Handled above. */
5418	/* SVM_EXIT_XCPT_BP: / / Handled above. */
5419	case SVM_EXIT_XCPT_OF:
5420	case SVM_EXIT_XCPT_BR:
5421	/* SVM_EXIT_XCPT_UD: / / Handled above. */
5422	case SVM_EXIT_XCPT_NM:
5423	case SVM_EXIT_XCPT_DF:
5424	case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5425	case SVM_EXIT_XCPT_TS:
5426	case SVM_EXIT_XCPT_NP:
5427	case SVM_EXIT_XCPT_SS:
5428	/* SVM_EXIT_XCPT_GP: / / Handled above. */
5429	/* SVM_EXIT_XCPT_PF: */
5430	case SVM_EXIT_XCPT_15: /* Reserved. */
5431	/* SVM_EXIT_XCPT_MF: / / Handled above. */
5432	/* SVM_EXIT_XCPT_AC: / / Handled above. */
5433	case SVM_EXIT_XCPT_MC:
5434	case SVM_EXIT_XCPT_XF:
5435	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5436	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5437	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5438	VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5439
5440	case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5441	case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5442	case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5443
5444	default:
5445	{
5446	AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5447	pVCpu->hm.s.u32HMError = uExitCode;
5448	return VERR_SVM_UNKNOWN_EXIT;
5449	}
5450	}
5451	}
5452	}
5453	/* not reached */
5454	}
5455
5456
5457	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5458	*
5459	* The following few functions and associated structure contains the bloat
5460	* necessary for providing detailed debug events and dtrace probes as well as
5461	* reliable host side single stepping. This works on the principle of
5462	* "subclassing" the normal execution loop and workers. We replace the loop
5463	* method completely and override selected helpers to add necessary adjustments
5464	* to their core operation.
5465	*
5466	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5467	* any performance for debug and analysis features.
5468	*
5469	* @{
5470	*/
5471
5472	/**
5473	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
5474	* the debug run loop.
5475	*/
5476	typedef struct SVMRUNDBGSTATE
5477	{
5478	/** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5479	uint64_t bmInterceptInitial;
5480	/** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5481	uint32_t bmXcptInitial;
5482	/** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5483	uint16_t bmInterceptRdCRxInitial;
5484	/** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5485	uint16_t bmInterceptWrCRxInitial;
5486	/** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5487	uint16_t bmInterceptRdDRxInitial;
5488	/** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5489	uint16_t bmInterceptWrDRxInitial;
5490
5491	/** Whether we've actually modified the intercept control qword. */
5492	bool fModifiedInterceptCtrl : 1;
5493	/** Whether we've actually modified the exception bitmap. */
5494	bool fModifiedXcptBitmap : 1;
5495	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5496	bool fModifiedInterceptRdCRx : 1;
5497	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5498	bool fModifiedInterceptWrCRx : 1;
5499	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5500	bool fModifiedInterceptRdDRx : 1;
5501	/** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5502	bool fModifiedInterceptWrDRx : 1;
5503
5504	/** The CS we started executing with. */
5505	uint16_t uCsStart;
5506	/** The RIP we started executing at. This is for detecting that we stepped. */
5507	uint64_t uRipStart;
5508
5509	/** The sequence number of the Dtrace provider settings the state was
5510	* configured against. */
5511	uint32_t uDtraceSettingsSeqNo;
5512	/** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5513	uint32_t bmXcptExtra;
5514	/** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5515	uint64_t bmInterceptExtra;
5516	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5517	uint16_t bmInterceptRdCRxExtra;
5518	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5519	uint16_t bmInterceptWrCRxExtra;
5520	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5521	uint16_t bmInterceptRdDRxExtra;
5522	/** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5523	uint16_t bmInterceptWrDRxExtra;
5524	/** VM-exits to check (one bit per VM-exit). */
5525	uint32_t bmExitsToCheck[33];
5526	} SVMRUNDBGSTATE;
5527	AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5528	typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5529
5530
5531	/**
5532	* Initializes the SVMRUNDBGSTATE structure.
5533	*
5534	* @param pVCpu The cross context virtual CPU structure of the
5535	* calling EMT.
5536	* @param pSvmTransient The SVM-transient structure.
5537	* @param pDbgState The debug state to initialize.
5538	*/
5539	static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5540	{
5541	PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5542	pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5543	pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5544	pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5545	pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5546	pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5547	pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5548
5549	pDbgState->fModifiedInterceptCtrl = false;
5550	pDbgState->fModifiedXcptBitmap = false;
5551	pDbgState->fModifiedInterceptRdCRx = false;
5552	pDbgState->fModifiedInterceptWrCRx = false;
5553	pDbgState->fModifiedInterceptRdDRx = false;
5554	pDbgState->fModifiedInterceptWrDRx = false;
5555
5556	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5557	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5558
5559	/* We don't really need to zero these. */
5560	pDbgState->bmInterceptExtra = 0;
5561	pDbgState->bmXcptExtra = 0;
5562	pDbgState->bmInterceptRdCRxExtra = 0;
5563	pDbgState->bmInterceptWrCRxExtra = 0;
5564	pDbgState->bmInterceptRdDRxExtra = 0;
5565	pDbgState->bmInterceptWrDRxExtra = 0;
5566	}
5567
5568
5569	/**
5570	* Updates the VMCB fields with changes requested by @a pDbgState.
5571	*
5572	* This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5573	* immediately before executing guest code, i.e. when interrupts are disabled.
5574	* We don't check status codes here as we cannot easily assert or return in the
5575	* latter case.
5576	*
5577	* @param pSvmTransient The SVM-transient structure.
5578	* @param pDbgState The debug state.
5579	*/
5580	static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5581	{
5582	/*
5583	* Ensure desired flags in VMCS control fields are set.
5584	*/
5585	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5586	#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5587	if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5588	{ \
5589	pVmcb->ctrl. a_VmcbCtrlField \|= (a_bmExtra); \
5590	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5591	Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5592	(a_fModified) = true; \
5593	} \
5594	} while (0)
5595	ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5596	ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5597	ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5598	ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5599	ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5600	ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5601	#undef ADD_EXTRA_INTERCEPTS
5602	}
5603
5604
5605	/**
5606	* Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5607	* for re-entry next time around.
5608	*
5609	* @returns Strict VBox status code (i.e. informational status codes too).
5610	* @param pSvmTransient The SVM-transient structure.
5611	* @param pDbgState The debug state.
5612	*/
5613	static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5614	{
5615	/*
5616	* Restore VM-exit control settings as we may not reenter this function the
5617	* next time around.
5618	*/
5619	PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5620
5621	#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5622	if ((a_fModified)) \
5623	{ \
5624	pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5625	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5626	} \
5627	} while (0)
5628	RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5629	RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5630	RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5631	RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5632	RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5633	RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5634	#undef RESTORE_INTERCEPTS
5635	}
5636
5637
5638	/**
5639	* Configures VM-exit controls for current DBGF and DTrace settings.
5640	*
5641	* This updates @a pDbgState and the VMCB execution control fields (in the debug
5642	* state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5643	*
5644	* @param pVCpu The cross context virtual CPU structure.
5645	* @param pSvmTransient The SVM-transient structure. May update
5646	* fUpdatedTscOffsettingAndPreemptTimer.
5647	* @param pDbgState The debug state.
5648	*/
5649	static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5650	{
5651	/*
5652	* Take down the dtrace serial number so we can spot changes.
5653	*/
5654	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5655	ASMCompilerBarrier();
5656
5657	/*
5658	* Clear data members that we'll be rebuilding here.
5659	*/
5660	pDbgState->bmXcptExtra = 0;
5661	pDbgState->bmInterceptExtra = 0;
5662	pDbgState->bmInterceptRdCRxExtra = 0;
5663	pDbgState->bmInterceptWrCRxExtra = 0;
5664	pDbgState->bmInterceptRdDRxExtra = 0;
5665	pDbgState->bmInterceptWrDRxExtra = 0;
5666	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5667	pDbgState->bmExitsToCheck[i] = 0;
5668
5669	/*
5670	* Software interrupts (INT XXh)
5671	*/
5672	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5673	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5674	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
5675	{
5676	pDbgState->bmInterceptExtra \|= SVM_CTRL_INTERCEPT_INTN;
5677	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5678	}
5679
5680	/*
5681	* INT3 breakpoints - triggered by #BP exceptions.
5682	*/
5683	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5684	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
5685
5686	/*
5687	* Exception bitmap and XCPT events+probes.
5688	*/
5689	#define SET_XCPT(a_iXcpt) do { \
5690	pDbgState->bmXcptExtra \|= RT_BIT_32(a_iXcpt); \
5691	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5692	} while (0)
5693
5694	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5695	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5696	SET_XCPT(iXcpt);
5697
5698	if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5699	if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5700	if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5701	if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5702	if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5703	if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5704	if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5705	if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5706	if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5707	if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5708	if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5709	if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5710	if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5711	if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5712	if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5713	if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5714	if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5715	if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5716
5717	#undef SET_XCPT
5718
5719	/*
5720	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5721	*
5722	* Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5723	* So, when adding/changing/removing please don't forget to update it.
5724	*
5725	* Some of the macros are picking up local variables to save horizontal space,
5726	* (being able to see it in a table is the lesser evil here).
5727	*/
5728	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5729	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5730	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5731	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5732	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5733	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5734	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5735	} else do { } while (0)
5736	#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5737	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5738	{ \
5739	(pDbgState)->bmInterceptExtra \|= (a_fInterceptCtrl); \
5740	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5741	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5742	} else do { } while (0)
5743
5744	/** @todo double check these */
5745	/** @todo Check what more AMD-V specific we can intercept. */
5746	//SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5747	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5748	SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5749	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5750	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5751	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5752	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5753	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5754	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5755	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5756	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5757	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5758	SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5759	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5760
5761	SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5762	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5763	SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5764	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5765	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5766	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5767	SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5768	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5769	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5770	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5771	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5772	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5773	SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5774	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5775	SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5776	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5777
5778	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5779	pDbgState->bmInterceptRdCRxExtra = 0xffff;
5780	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5781	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5782
5783	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5784	pDbgState->bmInterceptWrCRxExtra = 0xffff;
5785	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5786	{
5787	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5788	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5789	}
5790
5791	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5792	pDbgState->bmInterceptRdDRxExtra = 0xffff;
5793	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5794	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5795
5796	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5797	pDbgState->bmInterceptWrDRxExtra = 0xffff;
5798	if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) \|\| IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5799	ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5800
5801	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5802	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5803	SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5804	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5805	SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5806	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5807	if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5808	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5809	SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5810	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5811	SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5812	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5813	SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5814	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5815	SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5816	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5817	SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5819	SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5820	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5821	SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5822	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5823	SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5824	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5825	SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5826	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5827	SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5828	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5829	SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5831	SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5832	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5833
5834	if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5835	ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5836
5837	#undef IS_EITHER_ENABLED
5838	#undef SET_ONLY_XBM_IF_EITHER_EN
5839	#undef SET_INCP_XBM_IF_EITHER_EN
5840
5841	/*
5842	* Sanitize the control stuff.
5843	*/
5844	/** @todo filter out unsupported stuff? */
5845	if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5846	!= RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC \| SVM_CTRL_INTERCEPT_RDTSCP)))
5847	{
5848	pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5849	/// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5850	RT_NOREF(pSvmTransient);
5851	}
5852
5853	Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5854	pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5855	pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5856	pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5857	pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5858	pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5859	pDbgState->bmExitsToCheck[0],
5860	pDbgState->bmExitsToCheck[1],
5861	pDbgState->bmExitsToCheck[2],
5862	pDbgState->bmExitsToCheck[3],
5863	pDbgState->bmExitsToCheck[4]));
5864	}
5865
5866
5867	/**
5868	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
5869	* appropriate.
5870	*
5871	* The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5872	* bitmap.
5873	*
5874	* @returns Strict VBox status code (i.e. informational status codes too).
5875	* @param pVCpu The cross context virtual CPU structure.
5876	* @param pSvmTransient The SVM-transient structure.
5877	* @param uExitCode The VM-exit code.
5878	*
5879	* @remarks The name of this function is displayed by dtrace, so keep it short
5880	* and to the point. No longer than 33 chars long, please.
5881	*/
5882	static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5883	{
5884	/*
5885	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5886	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
5887	*
5888	* Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5889	* does. Must add/change/remove both places. Same ordering, please.
5890	*
5891	* Added/removed events must also be reflected in the next section
5892	* where we dispatch dtrace events.
5893	*/
5894	bool fDtrace1 = false;
5895	bool fDtrace2 = false;
5896	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5897	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5898	uint64_t uEventArg = 0;
5899	#define SET_XCPT(a_XcptName) \
5900	do { \
5901	enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5902	fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5903	} while (0)
5904	#define SET_EXIT(a_EventSubName) \
5905	do { \
5906	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5907	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5908	} while (0)
5909	#define SET_BOTH(a_EventSubName) \
5910	do { \
5911	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5912	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5913	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5914	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5915	} while (0)
5916	switch (uExitCode)
5917	{
5918	case SVM_EXIT_SWINT:
5919	enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5920	fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5921	uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5922	break;
5923
5924	case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5925	case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5926	case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5927	case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5928	case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5929	case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5930	case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5931	case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5932	case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5933	case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5934	case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5935	case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5936	case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5937	case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5938	case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5939	case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5940	case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5941	case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5942
5943	case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5944	case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5945	case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5946	case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5947	case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5948	case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5949	case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5950	case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5951	case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5952	case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5953	case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5954	case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5955	case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5956	case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5957
5958	case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5959	case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5960	case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5961	case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5962	case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5963	case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5964	case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5965	case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5966	case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5967	case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5968	case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5969	case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5970	case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5971	case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5972	case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5973
5974	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5975	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5976	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5977	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5978	SET_BOTH(CRX_READ);
5979	uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5980	break;
5981	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5982	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5983	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5984	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
5985	case SVM_EXIT_CR0_SEL_WRITE:
5986	SET_BOTH(CRX_WRITE);
5987	uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
5988	break;
5989	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5990	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
5991	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
5992	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5993	SET_BOTH(DRX_READ);
5994	uEventArg = uExitCode - SVM_EXIT_READ_DR0;
5995	break;
5996	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5997	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
5998	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
5999	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6000	SET_BOTH(DRX_WRITE);
6001	uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6002	break;
6003	case SVM_EXIT_MSR:
6004	if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6005	SET_BOTH(WRMSR);
6006	else
6007	SET_BOTH(RDMSR);
6008	break;
6009	case SVM_EXIT_MWAIT_ARMED:
6010	case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6011	case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6012	case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6013	case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6014	case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6015	case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6016	case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6017	case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6018	case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6019	case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6020	case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6021	case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6022	case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6023
6024	case SVM_EXIT_SHUTDOWN:
6025	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6026	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6027	break;
6028
6029	default:
6030	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6031	break;
6032	}
6033	#undef SET_BOTH
6034	#undef SET_EXIT
6035
6036	/*
6037	* Dtrace tracepoints go first. We do them here at once so we don't
6038	* have to copy the guest state saving and stuff a few dozen times.
6039	* Down side is that we've got to repeat the switch, though this time
6040	* we use enmEvent since the probes are a subset of what DBGF does.
6041	*/
6042	if (fDtrace1 \|\| fDtrace2)
6043	{
6044	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6045	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6046	switch (enmEvent1)
6047	{
6048	/** @todo consider which extra parameters would be helpful for each probe. */
6049	case DBGFEVENT_END: break;
6050	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6051	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6052	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6053	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6054	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6055	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6056	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6057	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6058	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6059	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6060	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6061	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6062	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6063	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6064	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6065	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6066	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6067	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6068	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6069	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6070	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6071	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6072	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6073	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6074	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6075	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6076	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6077	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6078	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6079	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6080	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6081	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6082	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6083	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6084	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6085	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6086	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6087	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6088	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6089	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6090	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6091	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6092	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6093	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6094	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6095	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6096	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6097	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6098	case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6099	case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6100	case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6101	case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6102	case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6103	default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6104	}
6105	switch (enmEvent2)
6106	{
6107	/** @todo consider which extra parameters would be helpful for each probe. */
6108	case DBGFEVENT_END: break;
6109	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6110	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6111	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6112	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6113	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6114	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6115	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6116	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6117	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6118	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6119	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6120	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6121	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6122	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6123	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6124	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6125	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6126	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6127	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6128	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6129	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6130	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6131	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6132	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6133	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6134	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6135	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6136	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6137	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6138	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6139	case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6140	case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6141	case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6142	case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6143	case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6144	default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6145	}
6146	}
6147
6148	/*
6149	* Fire of the DBGF event, if enabled (our check here is just a quick one,
6150	* the DBGF call will do a full check).
6151	*
6152	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6153	* Note! If we have to events, we prioritize the first, i.e. the instruction
6154	* one, in order to avoid event nesting.
6155	*/
6156	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6157	VBOXSTRICTRC rcStrict;
6158	if ( enmEvent1 != DBGFEVENT_END
6159	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6160	{
6161	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6162	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6163	}
6164	else if ( enmEvent2 != DBGFEVENT_END
6165	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6166	{
6167	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6168	rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6169	}
6170	else
6171	rcStrict = VINF_SUCCESS;
6172	return rcStrict;
6173	}
6174
6175
6176	/**
6177	* Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6178	* debug variant.
6179	*
6180	* @returns Strict VBox status code (informational status codes included).
6181	* @param pVCpu The cross context virtual CPU structure.
6182	* @param pSvmTransient Pointer to the SVM transient structure.
6183	* @param pDbgState The runtime debug state.
6184	*/
6185	static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6186	{
6187	Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6188	Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6189
6190	/*
6191	* Expensive (saves context) generic dtrace VM-exit probe.
6192	*/
6193	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6194	if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6195	{ /* more likely */ }
6196	else
6197	{
6198	hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6199	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6200	}
6201
6202	/*
6203	* Check for single stepping event if we're stepping.
6204	*/
6205	if (pVCpu->hm.s.fSingleInstruction)
6206	{
6207	switch (uExitCode)
6208	{
6209	/* Various events: */
6210	case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6211	case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6212	case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6213	case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6214	case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6215	case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6216	case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6217	case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6218	case SVM_EXIT_INTR:
6219	case SVM_EXIT_NMI:
6220	case SVM_EXIT_VINTR:
6221	case SVM_EXIT_NPF:
6222	case SVM_EXIT_AVIC_NOACCEL:
6223
6224	/* Instruction specific VM-exits: */
6225	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6226	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6227	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6228	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6229	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6230	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6231	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6232	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6233	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6234	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6235	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6236	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6237	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6238	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6239	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6240	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6241	case SVM_EXIT_CR0_SEL_WRITE:
6242	case SVM_EXIT_IDTR_READ:
6243	case SVM_EXIT_GDTR_READ:
6244	case SVM_EXIT_LDTR_READ:
6245	case SVM_EXIT_TR_READ:
6246	case SVM_EXIT_IDTR_WRITE:
6247	case SVM_EXIT_GDTR_WRITE:
6248	case SVM_EXIT_LDTR_WRITE:
6249	case SVM_EXIT_TR_WRITE:
6250	case SVM_EXIT_RDTSC:
6251	case SVM_EXIT_RDPMC:
6252	case SVM_EXIT_PUSHF:
6253	case SVM_EXIT_POPF:
6254	case SVM_EXIT_CPUID:
6255	case SVM_EXIT_RSM:
6256	case SVM_EXIT_IRET:
6257	case SVM_EXIT_SWINT:
6258	case SVM_EXIT_INVD:
6259	case SVM_EXIT_PAUSE:
6260	case SVM_EXIT_HLT:
6261	case SVM_EXIT_INVLPG:
6262	case SVM_EXIT_INVLPGA:
6263	case SVM_EXIT_IOIO:
6264	case SVM_EXIT_MSR:
6265	case SVM_EXIT_TASK_SWITCH:
6266	case SVM_EXIT_VMRUN:
6267	case SVM_EXIT_VMMCALL:
6268	case SVM_EXIT_VMLOAD:
6269	case SVM_EXIT_VMSAVE:
6270	case SVM_EXIT_STGI:
6271	case SVM_EXIT_CLGI:
6272	case SVM_EXIT_SKINIT:
6273	case SVM_EXIT_RDTSCP:
6274	case SVM_EXIT_ICEBP:
6275	case SVM_EXIT_WBINVD:
6276	case SVM_EXIT_MONITOR:
6277	case SVM_EXIT_MWAIT:
6278	case SVM_EXIT_MWAIT_ARMED:
6279	case SVM_EXIT_XSETBV:
6280	case SVM_EXIT_RDPRU:
6281	case SVM_EXIT_WRITE_EFER_TRAP:
6282	case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6283	case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6284	case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6285	case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6286	case SVM_EXIT_MCOMMIT:
6287	{
6288	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6289	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6290	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6291	{
6292	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6293	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6294	return VINF_EM_DBG_STEPPED;
6295	}
6296	break;
6297	}
6298
6299	/* Errors and unexpected events: */
6300	case SVM_EXIT_FERR_FREEZE:
6301	case SVM_EXIT_SHUTDOWN:
6302	case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6303	break;
6304
6305	case SVM_EXIT_SMI:
6306	case SVM_EXIT_INIT:
6307	default:
6308	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6309	break;
6310	}
6311	}
6312
6313	/*
6314	* Check for debugger event breakpoints and dtrace probes.
6315	*/
6316	if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6317	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6318	{
6319	VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6320	if (rcStrict != VINF_SUCCESS)
6321	{
6322	Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6323	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6324	return rcStrict;
6325	}
6326	}
6327
6328	/*
6329	* Normal processing.
6330	*/
6331	return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6332	}
6333
6334
6335	/**
6336	* Runs the guest code using AMD-V in single step mode.
6337	*
6338	* @returns Strict VBox status code.
6339	* @param pVCpu The cross context virtual CPU structure.
6340	* @param pcLoops Pointer to the number of executed loops.
6341	*/
6342	static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6343	{
6344	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6345	Assert(pcLoops);
6346	Assert(*pcLoops <= cMaxResumeLoops);
6347
6348	SVMTRANSIENT SvmTransient;
6349	RT_ZERO(SvmTransient);
6350	SvmTransient.fUpdateTscOffsetting = true;
6351	SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6352
6353	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6354
6355	/* Set HMCPU indicators. */
6356	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6357	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
6358	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6359	pVCpu->hmr0.s.fUsingDebugLoop = true;
6360
6361	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6362	SVMRUNDBGSTATE DbgState;
6363	hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6364	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6365
6366	/*
6367	* The loop.
6368	*/
6369	VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6370	for (;;)
6371	{
6372	Assert(!HMR0SuspendPending());
6373	AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6374	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6375	(unsigned)RTMpCpuId(), *pcLoops));
6376	bool fStepping = pVCpu->hm.s.fSingleInstruction;
6377
6378	/* Set up VM-execution controls the next two can respond to. */
6379	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6380
6381	/* Preparatory work for running nested-guest code, this may force us to return to
6382	ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6383	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6384	rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6385	if (rc != VINF_SUCCESS)
6386	break;
6387
6388	/*
6389	* No longjmps to ring-3 from this point on!!!
6390	*
6391	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6392	* better than a kernel panic. This also disables flushing of the R0-logger instance.
6393	*/
6394	hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6395
6396	/* Override any obnoxious code in the above two calls. */
6397	hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6398	#if 0
6399	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6400	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6401	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6402	#endif
6403
6404	/*
6405	* Finally execute guest code.
6406	*/
6407	rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6408
6409	/* Restore any residual host-state and save any bits shared between host and guest
6410	into the guest-CPU state. Re-enables interrupts! */
6411	hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6412	#if 0
6413	Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6414	SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6415	SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6416	#endif
6417
6418	if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6419	&& SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6420	{ /* very likely*/ }
6421	else
6422	{
6423	if (rc == VINF_SUCCESS)
6424	rc = VERR_SVM_INVALID_GUEST_STATE;
6425	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6426	hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6427	return rc;
6428	}
6429
6430	/* Handle the #VMEXIT. */
6431	HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6432	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6433	VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6434	rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6435	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6436	if (rc != VINF_SUCCESS)
6437	break;
6438	if (++(*pcLoops) >= cMaxResumeLoops)
6439	{
6440	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6441	rc = VINF_EM_RAW_INTERRUPT;
6442	break;
6443	}
6444
6445	/*
6446	* Stepping: Did the RIP change, if so, consider it a single step.
6447	* Otherwise, make sure one of the TFs gets set.
6448	*/
6449	if (fStepping)
6450	{
6451	hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
6452	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6453	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6454	{
6455	Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6456	pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6457	rc = VINF_EM_DBG_STEPPED;
6458	break;
6459	}
6460	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6461	}
6462
6463	/*
6464	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
6465	* Revert the state changes afterware so we can drop intercepts no longer needed.
6466	*/
6467	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6468	{
6469	hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6470	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6471	}
6472	}
6473
6474	/*
6475	* Clear the X86_EFL_TF if necessary.
6476	*/
6477	if (pVCpu->hmr0.s.fClearTrapFlag)
6478	{
6479	pVCpu->hmr0.s.fClearTrapFlag = false;
6480	pCtx->eflags.Bits.u1TF = 0;
6481	}
6482
6483	/* Restore HMCPU indicators. */
6484	pVCpu->hmr0.s.fUsingDebugLoop = false;
6485	pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6486	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6487
6488	/* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6489	hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6490
6491	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6492	return rc;
6493	}
6494
6495	/** @} */
6496
6497	#undef VMEXIT_CALL_RET
6498
6499
6500	#ifdef VBOX_STRICT
6501	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6502	# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6503	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6504
6505	# define HMSVM_ASSERT_PREEMPT_CPUID() \
6506	do \
6507	{ \
6508	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6509	AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6510	} while (0)
6511
6512	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6513	do { \
6514	AssertPtr((a_pVCpu)); \
6515	AssertPtr((a_pSvmTransient)); \
6516	Assert(ASMIntAreEnabled()); \
6517	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6518	HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6519	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6520	HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6521	if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6522	HMSVM_ASSERT_PREEMPT_CPUID(); \
6523	} while (0)
6524	#else
6525	# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6526	do { \
6527	RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6528	} while (0)
6529	#endif
6530
6531
6532	/**
6533	* Gets the IEM exception flags for the specified SVM event.
6534	*
6535	* @returns The IEM exception flags.
6536	* @param pEvent Pointer to the SVM event.
6537	*
6538	* @remarks This function currently only constructs flags required for
6539	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6540	* and CR2 aspects of an exception are not included).
6541	*/
6542	static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6543	{
6544	uint8_t const uEventType = pEvent->n.u3Type;
6545	uint32_t fIemXcptFlags;
6546	switch (uEventType)
6547	{
6548	case SVM_EVENT_EXCEPTION:
6549	/*
6550	* Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6551	* See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6552	*/
6553	if (pEvent->n.u8Vector == X86_XCPT_BP)
6554	{
6555	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_BP_INSTR;
6556	break;
6557	}
6558	if (pEvent->n.u8Vector == X86_XCPT_OF)
6559	{
6560	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_OF_INSTR;
6561	break;
6562	}
6563	/** @todo How do we distinguish ICEBP \#DB from the regular one? */
6564	RT_FALL_THRU();
6565	case SVM_EVENT_NMI:
6566	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6567	break;
6568
6569	case SVM_EVENT_EXTERNAL_IRQ:
6570	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6571	break;
6572
6573	case SVM_EVENT_SOFTWARE_INT:
6574	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6575	break;
6576
6577	default:
6578	fIemXcptFlags = 0;
6579	AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6580	break;
6581	}
6582	return fIemXcptFlags;
6583	}
6584
6585
6586	/**
6587	* Handle a condition that occurred while delivering an event through the guest
6588	* IDT.
6589	*
6590	* @returns VBox status code (informational error codes included).
6591	* @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6592	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6593	* continue execution of the guest which will delivery the \#DF.
6594	* @retval VINF_EM_RESET if we detected a triple-fault condition.
6595	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6596	*
6597	* @param pVCpu The cross context virtual CPU structure.
6598	* @param pSvmTransient Pointer to the SVM transient structure.
6599	*
6600	* @remarks No-long-jump zone!!!
6601	*/
6602	static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6603	{
6604	/** @todo r=bird: Looks like this is called on many exits and we start by
6605	* loading CR2 on the offchance that we actually have work to do here.
6606	*
6607	* HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6608	* pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6609	*
6610	* Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6611	*/
6612	int rc = VINF_SUCCESS;
6613	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6614	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6615
6616	Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6617	pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6618	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6619
6620	/*
6621	* The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6622	* be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6623	*
6624	* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6625	*/
6626	if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6627	{
6628	IEMXCPTRAISE enmRaise;
6629	IEMXCPTRAISEINFO fRaiseInfo;
6630	bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6631	uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6632	if (fExitIsHwXcpt)
6633	{
6634	uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6635	uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6636	uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6637	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6638	}
6639	else
6640	{
6641	/*
6642	* If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6643	* then we end up here.
6644	*
6645	* If the event was:
6646	* - a software interrupt, we can re-execute the instruction which will
6647	* regenerate the event.
6648	* - an NMI, we need to clear NMI blocking and re-inject the NMI.
6649	* - a hardware exception or external interrupt, we re-inject it.
6650	*/
6651	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6652	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6653	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6654	else
6655	enmRaise = IEMXCPTRAISE_PREV_EVENT;
6656	}
6657
6658	switch (enmRaise)
6659	{
6660	case IEMXCPTRAISE_CURRENT_XCPT:
6661	case IEMXCPTRAISE_PREV_EVENT:
6662	{
6663	/* For software interrupts, we shall re-execute the instruction. */
6664	if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6665	{
6666	RTGCUINTPTR GCPtrFaultAddress = 0;
6667
6668	/* If we are re-injecting an NMI, clear NMI blocking. */
6669	if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6670	CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
6671
6672	/* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6673	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
6674	{
6675	pSvmTransient->fVectoringPF = true;
6676	Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6677	pVCpu->cpum.GstCtx.cr2));
6678	}
6679	else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6680	&& uIdtVector == X86_XCPT_PF)
6681	{
6682	/*
6683	* If the previous exception was a #PF, we need to recover the CR2 value.
6684	* This can't happen with shadow paging.
6685	*/
6686	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6687	}
6688
6689	/*
6690	* Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6691	* exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6692	*/
6693	Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6694	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6695	hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6696
6697	Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6698	pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6699	pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6700	}
6701	break;
6702	}
6703
6704	case IEMXCPTRAISE_REEXEC_INSTR:
6705	{
6706	Assert(rc == VINF_SUCCESS);
6707	break;
6708	}
6709
6710	case IEMXCPTRAISE_DOUBLE_FAULT:
6711	{
6712	/*
6713	* Determing a vectoring double #PF condition. Used later, when PGM evaluates
6714	* the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6715	* converted into a #DF.
6716	*/
6717	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6718	{
6719	Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6720	pSvmTransient->fVectoringDoublePF = true;
6721	Assert(rc == VINF_SUCCESS);
6722	}
6723	else
6724	{
6725	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6726	hmR0SvmSetPendingXcptDF(pVCpu);
6727	rc = VINF_HM_DOUBLE_FAULT;
6728	}
6729	break;
6730	}
6731
6732	case IEMXCPTRAISE_TRIPLE_FAULT:
6733	{
6734	rc = VINF_EM_RESET;
6735	break;
6736	}
6737
6738	case IEMXCPTRAISE_CPU_HANG:
6739	{
6740	rc = VERR_EM_GUEST_CPU_HANG;
6741	break;
6742	}
6743
6744	default:
6745	AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6746	}
6747	}
6748	Assert(rc == VINF_SUCCESS \|\| rc == VINF_HM_DOUBLE_FAULT \|\| rc == VINF_EM_RESET \|\| rc == VERR_EM_GUEST_CPU_HANG);
6749	return rc;
6750	}
6751
6752
6753	/**
6754	* Advances the guest RIP by the number of bytes specified in @a cb.
6755	*
6756	* @param pVCpu The cross context virtual CPU structure.
6757	* @param cb RIP increment value in bytes.
6758	*/
6759	DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6760	{
6761	pVCpu->cpum.GstCtx.rip += cb;
6762	CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
6763	/** @todo clear RF. */
6764	}
6765
6766
6767	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6768	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6769	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6770
6771	/** @name \#VMEXIT handlers.
6772	* @{
6773	*/
6774
6775	/**
6776	* \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6777	* signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6778	*/
6779	HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6780	{
6781	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6782
6783	if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6784	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6785	else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6787
6788	/*
6789	* AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6790	* signal -before- the timer fires if the current interrupt is our own timer or a some
6791	* other host interrupt. We also cannot examine what interrupt it is until the host
6792	* actually take the interrupt.
6793	*
6794	* Going back to executing guest code here unconditionally causes random scheduling
6795	* problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6796	*/
6797	return VINF_EM_RAW_INTERRUPT;
6798	}
6799
6800
6801	/**
6802	* \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6803	*/
6804	HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6805	{
6806	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6807
6808	VBOXSTRICTRC rcStrict;
6809	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6810	if (fSupportsNextRipSave)
6811	{
6812	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6813	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6814	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6815	rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6816	}
6817	else
6818	{
6819	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6820	rcStrict = IEMExecOne(pVCpu);
6821	}
6822
6823	if (rcStrict == VINF_IEM_RAISED_XCPT)
6824	{
6825	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6826	rcStrict = VINF_SUCCESS;
6827	}
6828	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6829	return rcStrict;
6830	}
6831
6832
6833	/**
6834	* \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6835	*/
6836	HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6837	{
6838	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6839
6840	VBOXSTRICTRC rcStrict;
6841	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6842	if (fSupportsNextRipSave)
6843	{
6844	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6845	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6846	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6847	rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6848	}
6849	else
6850	{
6851	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6852	rcStrict = IEMExecOne(pVCpu);
6853	}
6854
6855	if (rcStrict == VINF_IEM_RAISED_XCPT)
6856	{
6857	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6858	rcStrict = VINF_SUCCESS;
6859	}
6860	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6861	return rcStrict;
6862	}
6863
6864
6865	/**
6866	* \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6867	*/
6868	HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6869	{
6870	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6871
6872	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RCX);
6873	VBOXSTRICTRC rcStrict;
6874	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6875	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
6876	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6877	if (!pExitRec)
6878	{
6879	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6880	if (fSupportsNextRipSave)
6881	{
6882	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6883	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6884	rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6885	}
6886	else
6887	{
6888	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6889	rcStrict = IEMExecOne(pVCpu);
6890	}
6891
6892	if (rcStrict == VINF_IEM_RAISED_XCPT)
6893	{
6894	CPUM_ASSERT_NOT_EXTRN(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
6895	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6896	rcStrict = VINF_SUCCESS;
6897	}
6898	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6899	}
6900	else
6901	{
6902	/*
6903	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
6904	*/
6905	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6906
6907	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6908	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6909
6910	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6911
6912	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6913	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6914	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6915	}
6916	return rcStrict;
6917	}
6918
6919
6920	/**
6921	* \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6922	*/
6923	HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6924	{
6925	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6926
6927	VBOXSTRICTRC rcStrict;
6928	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6929	if (fSupportsNextRipSave)
6930	{
6931	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
6932	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6933	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6934	rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6935	}
6936	else
6937	{
6938	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6939	rcStrict = IEMExecOne(pVCpu);
6940	}
6941
6942	if (rcStrict == VINF_SUCCESS)
6943	pSvmTransient->fUpdateTscOffsetting = true;
6944	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6945	{
6946	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6947	rcStrict = VINF_SUCCESS;
6948	}
6949	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6950	return rcStrict;
6951	}
6952
6953
6954	/**
6955	* \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6956	*/
6957	HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6958	{
6959	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6960
6961	VBOXSTRICTRC rcStrict;
6962	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6963	if (fSupportsNextRipSave)
6964	{
6965	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_TSC_AUX);
6966	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6967	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6968	rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6969	}
6970	else
6971	{
6972	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6973	rcStrict = IEMExecOne(pVCpu);
6974	}
6975
6976	if (rcStrict == VINF_SUCCESS)
6977	pSvmTransient->fUpdateTscOffsetting = true;
6978	else if (rcStrict == VINF_IEM_RAISED_XCPT)
6979	{
6980	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6981	rcStrict = VINF_SUCCESS;
6982	}
6983	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6984	return rcStrict;
6985	}
6986
6987
6988	/**
6989	* \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6990	*/
6991	HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6992	{
6993	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6994
6995	VBOXSTRICTRC rcStrict;
6996	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6997	if (fSupportsNextRipSave)
6998	{
6999	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR4);
7000	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7001	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7002	rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7003	}
7004	else
7005	{
7006	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7007	rcStrict = IEMExecOne(pVCpu);
7008	}
7009
7010	if (rcStrict == VINF_IEM_RAISED_XCPT)
7011	{
7012	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7013	rcStrict = VINF_SUCCESS;
7014	}
7015	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7016	return rcStrict;
7017	}
7018
7019
7020	/**
7021	* \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7022	*/
7023	HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7024	{
7025	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7026	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7027
7028	VBOXSTRICTRC rcStrict;
7029	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7030	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7031	if ( fSupportsDecodeAssists
7032	&& fSupportsNextRipSave)
7033	{
7034	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7035	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7036	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7037	RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7038	rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7039	}
7040	else
7041	{
7042	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7043	rcStrict = IEMExecOne(pVCpu);
7044	}
7045
7046	if (rcStrict == VINF_IEM_RAISED_XCPT)
7047	{
7048	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7049	rcStrict = VINF_SUCCESS;
7050	}
7051	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7052	return VBOXSTRICTRC_VAL(rcStrict);
7053	}
7054
7055
7056	/**
7057	* \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7058	*/
7059	HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7060	{
7061	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7062
7063	VBOXSTRICTRC rcStrict;
7064	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7065	if (fSupportsNextRipSave)
7066	{
7067	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7068	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7069	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7070	rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7071	}
7072	else
7073	{
7074	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7075	rcStrict = IEMExecOne(pVCpu);
7076	}
7077
7078	if ( rcStrict == VINF_EM_HALT
7079	\|\| rcStrict == VINF_SUCCESS)
7080	rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7081	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7082	{
7083	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7084	rcStrict = VINF_SUCCESS;
7085	}
7086	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7087	if (rcStrict != VINF_SUCCESS)
7088	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7089	return VBOXSTRICTRC_VAL(rcStrict);;
7090	}
7091
7092
7093	/**
7094	* \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7095	*/
7096	HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7097	{
7098	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7099
7100	/*
7101	* If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7102	* segment override prefix is present (and thus use the default segment DS). Otherwise, a
7103	* segment override prefix or other prefixes might be used, in which case we fallback to
7104	* IEMExecOne() to figure out.
7105	*/
7106	VBOXSTRICTRC rcStrict;
7107	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7108	uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7109	if (cbInstr)
7110	{
7111	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_DS);
7112	rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7113	}
7114	else
7115	{
7116	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7117	rcStrict = IEMExecOne(pVCpu);
7118	}
7119
7120	if (rcStrict == VINF_IEM_RAISED_XCPT)
7121	{
7122	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7123	rcStrict = VINF_SUCCESS;
7124	}
7125	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7126	return rcStrict;
7127	}
7128
7129
7130	/**
7131	* \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7132	*/
7133	HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7134	{
7135	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7136
7137	VBOXSTRICTRC rcStrict;
7138	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7139	if (fSupportsNextRipSave)
7140	{
7141	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7142	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7143	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7144	rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7145	}
7146	else
7147	{
7148	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7149	rcStrict = IEMExecOne(pVCpu);
7150	}
7151
7152	if ( rcStrict == VINF_EM_HALT
7153	&& EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7154	rcStrict = VINF_SUCCESS;
7155	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7156	{
7157	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7158	rcStrict = VINF_SUCCESS;
7159	}
7160	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7161	return rcStrict;
7162	}
7163
7164
7165	/**
7166	* \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7167	* \#VMEXIT.
7168	*/
7169	HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7170	{
7171	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7172	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7173	return VINF_EM_RESET;
7174	}
7175
7176
7177	/**
7178	* \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7179	*/
7180	HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7181	{
7182	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7183	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7184	AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7185	pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7186	RT_NOREF(pVmcb);
7187	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7188	return VERR_SVM_UNEXPECTED_EXIT;
7189	}
7190
7191
7192	/**
7193	* \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7194	*/
7195	HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7196	{
7197	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7198
7199	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7200	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7201	#ifdef VBOX_WITH_STATISTICS
7202	switch (pSvmTransient->u64ExitCode)
7203	{
7204	case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7205	case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7206	case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7207	case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7208	case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7209	}
7210	#endif
7211
7212	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7213	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7214	if ( fSupportsDecodeAssists
7215	&& fSupportsNextRipSave)
7216	{
7217	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7218	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7219	if (fMovCRx)
7220	{
7221	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_CR_MASK
7222	\| CPUMCTX_EXTRN_APIC_TPR);
7223	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7224	uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7225	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7226	VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7227	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7228	return VBOXSTRICTRC_VAL(rcStrict);
7229	}
7230	/* else: SMSW instruction, fall back below to IEM for this. */
7231	}
7232
7233	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7234	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7235	AssertMsg( rcStrict == VINF_SUCCESS
7236	\|\| rcStrict == VINF_PGM_SYNC_CR3
7237	\|\| rcStrict == VINF_IEM_RAISED_XCPT,
7238	("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7239	Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7240	if (rcStrict == VINF_IEM_RAISED_XCPT)
7241	{
7242	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7243	rcStrict = VINF_SUCCESS;
7244	}
7245	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7246	return rcStrict;
7247	}
7248
7249
7250	/**
7251	* \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7252	*/
7253	HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7254	{
7255	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7256
7257	uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7258	uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7259	Assert(iCrReg <= 15);
7260
7261	VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7262	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7263	bool fDecodedInstr = false;
7264	bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7265	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7266	if ( fSupportsDecodeAssists
7267	&& fSupportsNextRipSave)
7268	{
7269	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7270	bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7271	if (fMovCRx)
7272	{
7273	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4
7274	\| CPUMCTX_EXTRN_APIC_TPR);
7275	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7276	uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7277	Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7278	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7279	fDecodedInstr = true;
7280	}
7281	/* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7282	}
7283
7284	if (!fDecodedInstr)
7285	{
7286	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7287	Log4Func(("iCrReg=%#x\n", iCrReg));
7288	rcStrict = IEMExecOne(pVCpu);
7289	if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7290	\|\| rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7291	rcStrict = VERR_EM_INTERPRETER;
7292	}
7293
7294	if (rcStrict == VINF_SUCCESS)
7295	{
7296	switch (iCrReg)
7297	{
7298	case 0:
7299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7301	break;
7302
7303	case 2:
7304	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7306	break;
7307
7308	case 3:
7309	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7311	break;
7312
7313	case 4:
7314	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7316	break;
7317
7318	case 8:
7319	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7321	break;
7322
7323	default:
7324	{
7325	AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7326	pSvmTransient->u64ExitCode, iCrReg));
7327	break;
7328	}
7329	}
7330	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7331	}
7332	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7333	{
7334	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7335	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7336	rcStrict = VINF_SUCCESS;
7337	}
7338	else
7339	Assert(rcStrict == VERR_EM_INTERPRETER \|\| rcStrict == VINF_PGM_SYNC_CR3);
7340	return rcStrict;
7341	}
7342
7343
7344	/**
7345	* \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7346	*
7347	* @returns Strict VBox status code.
7348	* @param pVCpu The cross context virtual CPU structure.
7349	* @param pVmcb Pointer to the VM control block.
7350	*/
7351	static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7352	{
7353	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7354	Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7355
7356	VBOXSTRICTRC rcStrict;
7357	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7358	if (fSupportsNextRipSave)
7359	{
7360	/** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7361	* can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7362	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7363	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7364	rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7365	}
7366	else
7367	{
7368	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7369	rcStrict = IEMExecOne(pVCpu);
7370	}
7371
7372	AssertMsg( rcStrict == VINF_SUCCESS
7373	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7374	\|\| rcStrict == VINF_CPUM_R3_MSR_READ,
7375	("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7376
7377	if (rcStrict == VINF_IEM_RAISED_XCPT)
7378	{
7379	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7380	rcStrict = VINF_SUCCESS;
7381	}
7382	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7383	return rcStrict;
7384	}
7385
7386
7387	/**
7388	* \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7389	*
7390	* @returns Strict VBox status code.
7391	* @param pVCpu The cross context virtual CPU structure.
7392	* @param pVmcb Pointer to the VM control block.
7393	* @param pSvmTransient Pointer to the SVM-transient structure.
7394	*/
7395	static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7396	{
7397	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7398	uint32_t const idMsr = pCtx->ecx;
7399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7400	Log4Func(("idMsr=%#RX32\n", idMsr));
7401
7402	/*
7403	* Handle TPR patching MSR writes.
7404	* We utilitize the LSTAR MSR for patching.
7405	*/
7406	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7407	if ( idMsr == MSR_K8_LSTAR
7408	&& pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7409	{
7410	unsigned cbInstr;
7411	if (fSupportsNextRipSave)
7412	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7413	else
7414	{
7415	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
7416	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
7417	if ( rc == VINF_SUCCESS
7418	&& pDis->pCurInstr->uOpcode == OP_WRMSR)
7419	Assert(cbInstr > 0);
7420	else
7421	cbInstr = 0;
7422	}
7423
7424	/* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7425	if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7426	{
7427	int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7428	AssertRCReturn(rc, rc);
7429	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7430	}
7431
7432	int rc = VINF_SUCCESS;
7433	hmR0SvmAdvanceRip(pVCpu, cbInstr);
7434	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7435	return rc;
7436	}
7437
7438	/*
7439	* Handle regular MSR writes.
7440	*/
7441	VBOXSTRICTRC rcStrict;
7442	if (fSupportsNextRipSave)
7443	{
7444	/** @todo Optimize this: We don't need to get much of the MSR state here
7445	* since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7446	* clear the applicable extern flags. */
7447	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7448	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7449	rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7450	}
7451	else
7452	{
7453	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_ALL_MSRS);
7454	rcStrict = IEMExecOne(pVCpu);
7455	}
7456
7457	AssertMsg( rcStrict == VINF_SUCCESS
7458	\|\| rcStrict == VINF_IEM_RAISED_XCPT
7459	\|\| rcStrict == VINF_CPUM_R3_MSR_WRITE,
7460	("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7461
7462	if (rcStrict == VINF_SUCCESS)
7463	{
7464	/* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7465	if ( idMsr >= MSR_IA32_X2APIC_START
7466	&& idMsr <= MSR_IA32_X2APIC_END)
7467	{
7468	/*
7469	* We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7470	* When full APIC register virtualization is implemented we'll have to make sure
7471	* APIC state is saved from the VMCB before IEM changes it.
7472	*/
7473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7474	}
7475	else
7476	{
7477	switch (idMsr)
7478	{
7479	case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7480	case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7481	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7482	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7483	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7484	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7485	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7486	}
7487	}
7488	}
7489	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7490	{
7491	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7492	rcStrict = VINF_SUCCESS;
7493	}
7494	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7495	return rcStrict;
7496	}
7497
7498
7499	/**
7500	* \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7501	* \#VMEXIT.
7502	*/
7503	HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7504	{
7505	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7506
7507	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7508	if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7509	return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7510
7511	Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7512	return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7513	}
7514
7515
7516	/**
7517	* \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7518	*/
7519	HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7520	{
7521	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7522	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7523
7524	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7525
7526	/** @todo Stepping with nested-guest. */
7527	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7528	if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7529	{
7530	/* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7531	if (pSvmTransient->fWasGuestDebugStateActive)
7532	{
7533	AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7534	pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7535	return VERR_SVM_UNEXPECTED_EXIT;
7536	}
7537
7538	/*
7539	* Lazy DR0-3 loading.
7540	*/
7541	if (!pSvmTransient->fWasHyperDebugStateActive)
7542	{
7543	Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7544	Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7545
7546	/* Don't intercept DRx read and writes. */
7547	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7548	pVmcb->ctrl.u16InterceptRdDRx = 0;
7549	pVmcb->ctrl.u16InterceptWrDRx = 0;
7550	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7551
7552	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7553	VMMRZCallRing3Disable(pVCpu);
7554	HM_DISABLE_PREEMPT(pVCpu);
7555
7556	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7557	CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7558	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7559
7560	HM_RESTORE_PREEMPT();
7561	VMMRZCallRing3Enable(pVCpu);
7562
7563	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7564	return VINF_SUCCESS;
7565	}
7566	}
7567
7568	/*
7569	* Interpret the read/writing of DRx.
7570	*/
7571	/** @todo Decode assist. */
7572	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
7573	Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7574	if (RT_LIKELY(rc == VINF_SUCCESS))
7575	{
7576	/* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7577	/** @todo CPUM should set this flag! */
7578	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7579	HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7580	}
7581	else
7582	Assert(rc == VERR_EM_INTERPRETER);
7583	return rc;
7584	}
7585
7586
7587	/**
7588	* \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7589	*/
7590	HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7591	{
7592	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7593	/* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7594	VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7596	STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7597	return rc;
7598	}
7599
7600
7601	/**
7602	* \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7603	*/
7604	HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7605	{
7606	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7607	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7608
7609	/** @todo decode assists... */
7610	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7611	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7612	{
7613	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7614	bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7615	Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7616	if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7617	{
7618	pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7619	hmR0SvmUpdateVmRunFunction(pVCpu);
7620	}
7621	}
7622	else if (rcStrict == VINF_IEM_RAISED_XCPT)
7623	{
7624	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7625	rcStrict = VINF_SUCCESS;
7626	}
7627	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7628	return rcStrict;
7629	}
7630
7631
7632	/**
7633	* \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7634	*/
7635	HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7636	{
7637	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7638	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK);
7639
7640	/* I/O operation lookup arrays. */
7641	static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7642	static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7643	the result (in AL/AX/EAX). */
7644	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7645	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7646	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7647
7648	Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7649
7650	/* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7651	SVMIOIOEXITINFO IoExitInfo;
7652	IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7653	uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7654	uint32_t cbValue = s_aIOSize[uIOWidth];
7655	uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7656
7657	if (RT_UNLIKELY(!cbValue))
7658	{
7659	AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7660	return VERR_EM_INTERPRETER;
7661	}
7662
7663	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
7664	VBOXSTRICTRC rcStrict;
7665	PCEMEXITREC pExitRec = NULL;
7666	if ( !pVCpu->hm.s.fSingleInstruction
7667	&& !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7668	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7669	!IoExitInfo.n.u1Str
7670	? IoExitInfo.n.u1Type == SVM_IOIO_READ
7671	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7672	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7673	: IoExitInfo.n.u1Type == SVM_IOIO_READ
7674	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7675	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7676	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7677	if (!pExitRec)
7678	{
7679	bool fUpdateRipAlready = false;
7680	if (IoExitInfo.n.u1Str)
7681	{
7682	/* INS/OUTS - I/O String instruction. */
7683	/** @todo Huh? why can't we use the segment prefix information given by AMD-V
7684	* in EXITINFO1? Investigate once this thing is up and running. */
7685	Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7686	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7687	AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7688	static IEMMODE const s_aenmAddrMode[8] =
7689	{
7690	(IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7691	};
7692	IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7693	if (enmAddrMode != (IEMMODE)-1)
7694	{
7695	uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7696	if (cbInstr <= 15 && cbInstr >= 1)
7697	{
7698	Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7699	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7700	{
7701	/* Don't know exactly how to detect whether u3Seg is valid, currently
7702	only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7703	2384 Opterons when only checking NRIP. */
7704	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7705	if ( fSupportsNextRipSave
7706	&& pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7707	{
7708	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS \|\| cbInstr > 1U + IoExitInfo.n.u1Rep,
7709	("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7710	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7711	IoExitInfo.n.u3Seg, true /fIoChecked/);
7712	}
7713	else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7714	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7715	X86_SREG_DS, true /fIoChecked/);
7716	else
7717	rcStrict = IEMExecOne(pVCpu);
7718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7719	}
7720	else
7721	{
7722	AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /=0/, ("%#x\n", IoExitInfo.n.u3Seg));
7723	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7724	true /fIoChecked/);
7725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7726	}
7727	}
7728	else
7729	{
7730	AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7731	rcStrict = IEMExecOne(pVCpu);
7732	}
7733	}
7734	else
7735	{
7736	AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7737	rcStrict = IEMExecOne(pVCpu);
7738	}
7739	fUpdateRipAlready = true;
7740	if (rcStrict == VINF_IEM_RAISED_XCPT)
7741	{
7742	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7743	rcStrict = VINF_SUCCESS;
7744	}
7745	}
7746	else
7747	{
7748	/* IN/OUT - I/O instruction. */
7749	Assert(!IoExitInfo.n.u1Rep);
7750
7751	uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7752	if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7753	{
7754	rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7755	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7756	&& !pCtx->eflags.Bits.u1TF)
7757	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7759	}
7760	else
7761	{
7762	uint32_t u32Val = 0;
7763	rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7764	if (IOM_SUCCESS(rcStrict))
7765	{
7766	/* Save result of I/O IN instr. in AL/AX/EAX. */
7767	/** @todo r=bird: 32-bit op size should clear high bits of rax! */
7768	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
7769	}
7770	else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7771	&& !pCtx->eflags.Bits.u1TF)
7772	rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7773
7774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7775	}
7776	}
7777
7778	if (IOM_SUCCESS(rcStrict))
7779	{
7780	/* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7781	if (!fUpdateRipAlready)
7782	pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7783
7784	/*
7785	* If any I/O breakpoints are armed, we need to check if one triggered
7786	* and take appropriate action.
7787	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7788	*/
7789	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7790	* execution engines about whether hyper BPs and such are pending. */
7791	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7792	uint32_t const uDr7 = pCtx->dr[7];
7793	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7794	&& X86_DR7_ANY_RW_IO(uDr7)
7795	&& (pCtx->cr4 & X86_CR4_DE))
7796	\|\| DBGFBpIsHwIoArmed(pVM)))
7797	{
7798	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7799	VMMRZCallRing3Disable(pVCpu);
7800	HM_DISABLE_PREEMPT(pVCpu);
7801
7802	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7803	CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /fDr6/);
7804
7805	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7806	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7807	{
7808	/* Raise #DB. */
7809	pVmcb->guest.u64DR6 = pCtx->dr[6];
7810	pVmcb->guest.u64DR7 = pCtx->dr[7];
7811	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7812	hmR0SvmSetPendingXcptDB(pVCpu);
7813	}
7814	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7815	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7816	else if ( rcStrict2 != VINF_SUCCESS
7817	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
7818	rcStrict = rcStrict2;
7819	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7820
7821	HM_RESTORE_PREEMPT();
7822	VMMRZCallRing3Enable(pVCpu);
7823	}
7824
7825	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7826	}
7827	#ifdef VBOX_STRICT
7828	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7829	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7830	Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7831	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7832	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7833	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7834	Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7835	else
7836	{
7837	/** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7838	* statuses, that the VMM device and some others may return. See
7839	* IOM_SUCCESS() for guidance. */
7840	AssertMsg( RT_FAILURE(rcStrict)
7841	\|\| rcStrict == VINF_SUCCESS
7842	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
7843	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
7844	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
7845	\|\| rcStrict == VINF_EM_DBG_STEPPED
7846	\|\| rcStrict == VINF_EM_RAW_TO_R3
7847	\|\| rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7848	}
7849	#endif
7850	}
7851	else
7852	{
7853	/*
7854	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7855	*/
7856	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7857	STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7858	? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7859	: IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7860	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7861	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7862	IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7863
7864	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7865	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7866
7867	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7868	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7869	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7870	}
7871	return rcStrict;
7872	}
7873
7874
7875	/**
7876	* \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7877	*/
7878	HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7879	{
7880	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7881	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7882	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7883
7884	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7885	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7886	Assert(pVM->hmr0.s.fNestedPaging);
7887
7888	/* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7889	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7890	RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7891	uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7892	thus intentionally not copied into u32ErrCode. */
7893
7894	Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7895	u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7896
7897	/*
7898	* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7899	*/
7900	if ( pVM->hm.s.fTprPatchingAllowed
7901	&& (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7902	&& ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7903	\|\| (u32ErrCode & (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P \| X86_TRAP_PF_RSVD)) /* MMIO page. */
7904	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7905	&& !CPUMIsGuestInLongModeEx(pCtx)
7906	&& !CPUMGetGuestCPL(pVCpu)
7907	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7908	{
7909	RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7910	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
7911
7912	if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7913	{
7914	/* Only attempt to patch the instruction once. */
7915	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7916	if (!pPatch)
7917	return VINF_EM_HM_PATCH_TPR_INSTR;
7918	}
7919	}
7920
7921	/*
7922	* Determine the nested paging mode.
7923	*/
7924	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7925	PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7926
7927	/*
7928	* MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7929	*/
7930	Assert((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7931	if ((u32ErrCode & (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD \| X86_TRAP_PF_P))
7932	{
7933	/*
7934	* If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7935	* injecting the original pending event would most likely cause the same MMIO #NPF.
7936	*/
7937	if (pVCpu->hm.s.Event.fPending)
7938	{
7939	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7940	return VINF_EM_RAW_INJECT_TRPM_EVENT;
7941	}
7942
7943	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
7944	VBOXSTRICTRC rcStrict;
7945	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7946	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
7947	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7948	if (!pExitRec)
7949	{
7950
7951	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, pCtx, GCPhysFaultAddr, u32ErrCode);
7952
7953	/*
7954	* If we succeed, resume guest execution.
7955	*
7956	* If we fail in interpreting the instruction because we couldn't get the guest
7957	* physical address of the page containing the instruction via the guest's page
7958	* tables (we would invalidate the guest page in the host TLB), resume execution
7959	* which would cause a guest page fault to let the guest handle this weird case.
7960	*
7961	* See @bugref{6043}.
7962	*/
7963	if ( rcStrict == VINF_SUCCESS
7964	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7965	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
7966	{
7967	/* Successfully handled MMIO operation. */
7968	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7969	rcStrict = VINF_SUCCESS;
7970	}
7971	}
7972	else
7973	{
7974	/*
7975	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
7976	*/
7977	Assert(pCtx == &pVCpu->cpum.GstCtx);
7978	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7979	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7980	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
7981
7982	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7983	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7984
7985	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7986	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7987	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7988	}
7989	return rcStrict;
7990	}
7991
7992	/*
7993	* Nested page-fault.
7994	*/
7995	TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
7996	int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, pCtx, GCPhysFaultAddr);
7997	TRPMResetTrap(pVCpu);
7998
7999	Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8000
8001	/*
8002	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8003	*/
8004	if ( rc == VINF_SUCCESS
8005	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8006	\|\| rc == VERR_PAGE_NOT_PRESENT)
8007	{
8008	/* We've successfully synced our shadow page tables. */
8009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8010	rc = VINF_SUCCESS;
8011	}
8012
8013	/*
8014	* If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8015	* re-inject the original event.
8016	*/
8017	if (pVCpu->hm.s.Event.fPending)
8018	{
8019	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8020
8021	/*
8022	* If the #NPF handler requested emulation of the instruction, ignore it.
8023	* We need to re-inject the original event so as to not lose it.
8024	* Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8025	* LiveCD is broken for other reasons).
8026	*/
8027	if (rc == VINF_EM_RAW_EMULATE_INSTR)
8028	rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8029	}
8030
8031	return rc;
8032	}
8033
8034
8035	/**
8036	* \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8037	* \#VMEXIT.
8038	*/
8039	HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8040	{
8041	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8042	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8043
8044	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8045	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8046	hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8047
8048	/* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8049	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8050	return VINF_SUCCESS;
8051	}
8052
8053
8054	/**
8055	* \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8056	* \#VMEXIT.
8057	*/
8058	HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8059	{
8060	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8061	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8062
8063	#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8064	Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8065	#endif
8066
8067	/* Check if this task-switch occurred while delivering an event through the guest IDT. */
8068	if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8069	{
8070	/*
8071	* AMD-V provides us with the exception which caused the TS; we collect
8072	* the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8073	*/
8074	Log4Func(("TS occurred during event delivery\n"));
8075	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8076	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8077	}
8078
8079	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8080	* emulation. */
8081	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8082	return VERR_EM_INTERPRETER;
8083	}
8084
8085
8086	/**
8087	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8088	*/
8089	HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8090	{
8091	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8092	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8093
8094	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8095	if (pVM->hm.s.fTprPatchingAllowed)
8096	{
8097	int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8098	if (rc != VERR_NOT_FOUND)
8099	{
8100	Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8101	return rc;
8102	}
8103	}
8104
8105	if (EMAreHypercallInstructionsEnabled(pVCpu))
8106	{
8107	unsigned cbInstr;
8108	if (hmR0SvmSupportsNextRipSave(pVCpu))
8109	{
8110	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8111	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8112	}
8113	else
8114	{
8115	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8116	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8117	if ( rc == VINF_SUCCESS
8118	&& pDis->pCurInstr->uOpcode == OP_VMMCALL)
8119	Assert(cbInstr > 0);
8120	else
8121	cbInstr = 0;
8122	}
8123
8124	VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8125	if (RT_SUCCESS(rcStrict))
8126	{
8127	/* Only update the RIP if we're continuing guest execution and not in the case
8128	of say VINF_GIM_R3_HYPERCALL. */
8129	if (rcStrict == VINF_SUCCESS)
8130	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8131
8132	return VBOXSTRICTRC_VAL(rcStrict);
8133	}
8134	else
8135	Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8136	}
8137
8138	hmR0SvmSetPendingXcptUD(pVCpu);
8139	return VINF_SUCCESS;
8140	}
8141
8142
8143	/**
8144	* \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8145	*/
8146	HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8147	{
8148	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8149
8150	unsigned cbInstr;
8151	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8152	if (fSupportsNextRipSave)
8153	{
8154	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8155	cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8156	}
8157	else
8158	{
8159	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8160	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8161	if ( rc == VINF_SUCCESS
8162	&& pDis->pCurInstr->uOpcode == OP_PAUSE)
8163	Assert(cbInstr > 0);
8164	else
8165	cbInstr = 0;
8166	}
8167
8168	/** @todo The guest has likely hit a contended spinlock. We might want to
8169	* poke a schedule different guest VCPU. */
8170	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8171	return VINF_EM_RAW_INTERRUPT;
8172	}
8173
8174
8175	/**
8176	* \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8177	* \#VMEXIT.
8178	*/
8179	HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8180	{
8181	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8182	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8183	Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8184
8185	Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8186	return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8187	}
8188
8189
8190	/**
8191	* \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8192	*/
8193	HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8194	{
8195	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8196
8197	/* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8198	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8199	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8200
8201	/* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8202	* deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8203	* handler executes. Yes, x86 is ugly.
8204	*/
8205	return VINF_EM_RAW_EMULATE_INSTR;
8206	}
8207
8208
8209	/**
8210	* \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8211	* Conditional \#VMEXIT.
8212	*/
8213	HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8214	{
8215	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8216	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8217	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8218
8219	/* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8220	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8221	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8222	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8223	uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8224	uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8225
8226	#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMSVM_ALWAYS_TRAP_PF)
8227	if (pVM->hmr0.s.fNestedPaging)
8228	{
8229	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8230	if ( !pSvmTransient->fVectoringDoublePF
8231	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8232	{
8233	/* A genuine guest #PF, reflect it to the guest. */
8234	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8235	Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8236	uFaultAddress, uErrCode));
8237	}
8238	else
8239	{
8240	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8241	hmR0SvmSetPendingXcptDF(pVCpu);
8242	Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8243	}
8244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8245	return VINF_SUCCESS;
8246	}
8247	#endif
8248
8249	Assert(!pVM->hmr0.s.fNestedPaging);
8250
8251	/*
8252	* TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8253	*/
8254	if ( pVM->hm.s.fTprPatchingAllowed
8255	&& (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8256	&& !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8257	&& !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8258	&& !CPUMIsGuestInLongModeEx(pCtx)
8259	&& !CPUMGetGuestCPL(pVCpu)
8260	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8261	{
8262	RTGCPHYS GCPhysApicBase;
8263	GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8264	GCPhysApicBase &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
8265
8266	/* Check if the page at the fault-address is the APIC base. */
8267	PGMPTWALK Walk;
8268	int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8269	if ( rc2 == VINF_SUCCESS
8270	&& Walk.GCPhys == GCPhysApicBase)
8271	{
8272	/* Only attempt to patch the instruction once. */
8273	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8274	if (!pPatch)
8275	return VINF_EM_HM_PATCH_TPR_INSTR;
8276	}
8277	}
8278
8279	Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8280	pCtx->rip, uErrCode, pCtx->cr3));
8281
8282	/*
8283	* If it's a vectoring #PF, emulate injecting the original event injection as
8284	* PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8285	* event injection that caused a #PF. See @bugref{6607}.
8286	*/
8287	if (pSvmTransient->fVectoringPF)
8288	{
8289	Assert(pVCpu->hm.s.Event.fPending);
8290	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8291	}
8292
8293	TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8294	int rc = PGMTrap0eHandler(pVCpu, uErrCode, pCtx, (RTGCPTR)uFaultAddress);
8295
8296	Log4Func(("#PF: rc=%Rrc\n", rc));
8297
8298	if (rc == VINF_SUCCESS)
8299	{
8300	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8301	TRPMResetTrap(pVCpu);
8302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8304	return rc;
8305	}
8306
8307	if (rc == VINF_EM_RAW_GUEST_TRAP)
8308	{
8309	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8310
8311	/*
8312	* If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8313	* we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8314	* determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8315	*/
8316	if ( !pSvmTransient->fVectoringDoublePF
8317	\|\| CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8318	{
8319	/* It's a guest (or nested-guest) page fault and needs to be reflected. */
8320	uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8321	TRPMResetTrap(pVCpu);
8322
8323	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8324	/* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8325	if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8326	&& CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8327	return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8328	#endif
8329
8330	hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8331	}
8332	else
8333	{
8334	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8335	TRPMResetTrap(pVCpu);
8336	hmR0SvmSetPendingXcptDF(pVCpu);
8337	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8338	}
8339
8340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8341	return VINF_SUCCESS;
8342	}
8343
8344	TRPMResetTrap(pVCpu);
8345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8346	return rc;
8347	}
8348
8349
8350
8351	/**
8352	* \#VMEXIT handler for division overflow exceptions (SVM_EXIT_XCPT_1).
8353	* Conditional \#VMEXIT.
8354	*/
8355	HMSVM_EXIT_DECL hmR0SvmExitXcptDE(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8356	{
8357	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8358	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8360
8361	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8362	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8363	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8364
8365	int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8366	if (pVCpu->hm.s.fGCMTrapXcptDE)
8367	{
8368	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8369	uint8_t cbInstr = 0;
8370	VBOXSTRICTRC rcStrict = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8371	if (rcStrict == VINF_SUCCESS)
8372	rc = VINF_SUCCESS; /* Restart instruction with modified guest register context. */
8373	else if (rcStrict == VERR_NOT_FOUND)
8374	rc = VERR_NOT_FOUND; /* Deliver the exception. */
8375	else
8376	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8377	}
8378
8379	/* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
8380	if (RT_FAILURE(rc))
8381	{
8382	hmR0SvmSetPendingXcptDE(pVCpu);
8383	rc = VINF_SUCCESS;
8384	}
8385
8386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8387	return rc;
8388	}
8389
8390
8391	/**
8392	* \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8393	* Conditional \#VMEXIT.
8394	*/
8395	HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8396	{
8397	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8398	HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8400
8401	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8402	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8403	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8404
8405	/** @todo if we accumulate more optional stuff here, we ought to combine the
8406	* reading of opcode bytes to avoid doing more than once. */
8407
8408	VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8409	if (pVCpu->hm.s.fGIMTrapXcptUD)
8410	{
8411	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8412	uint8_t cbInstr = 0;
8413	rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8414	if (rcStrict == VINF_SUCCESS)
8415	{
8416	/* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8417	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8418	rcStrict = VINF_SUCCESS;
8419	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8420	}
8421	else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8422	rcStrict = VINF_SUCCESS;
8423	else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8424	rcStrict = VINF_GIM_R3_HYPERCALL;
8425	else
8426	{
8427	Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8428	rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8429	}
8430	}
8431
8432	if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8433	{
8434	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS
8435	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8436	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8437	{
8438	/* Ideally, IEM should just handle all these special #UD situations, but
8439	we don't quite trust things to behave optimially when doing that. So,
8440	for now we'll restrict ourselves to a handful of possible sysenter and
8441	sysexit encodings that we filter right here. */
8442	uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8443	uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8444	uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8445	uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8446	RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8447	if (cbInstr < cbMin \|\| cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8448	{
8449	cbInstr = cbMin;
8450	int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8451	AssertRCStmt(rc2, cbInstr = 0);
8452	}
8453	else
8454	memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8455	if ( cbInstr == 0 /* read error */
8456	\|\| (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8457	\|\| ( uCpl == 0
8458	&& ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8459	\|\| ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8460	&& (abInstr[0] & (X86_OP_REX_W \| 0xf0)) == X86_OP_REX_W))))
8461	{
8462	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8463	\| CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8464	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8465	rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, GCPtrInstr, abInstr, cbInstr);
8466	Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8467	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8468	pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8469	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8470	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8471	if (rcStrict == VINF_IEM_RAISED_XCPT)
8472	rcStrict = VINF_SUCCESS;
8473	return rcStrict;
8474	}
8475	Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8476	}
8477	else
8478	Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8479	}
8480
8481	/* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8482	if (RT_FAILURE(rcStrict))
8483	{
8484	hmR0SvmSetPendingXcptUD(pVCpu);
8485	rcStrict = VINF_SUCCESS;
8486	}
8487
8488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8489	return rcStrict;
8490	}
8491
8492
8493	/**
8494	* \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8495	* Conditional \#VMEXIT.
8496	*/
8497	HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8498	{
8499	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8500	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8501	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8502
8503	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8504	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8505
8506	/* Paranoia; Ensure we cannot be called as a result of event delivery. */
8507	Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8508
8509	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8510
8511	if (!(pCtx->cr0 & X86_CR0_NE))
8512	{
8513	PDISSTATE pDis = &pVCpu->hmr0.s.svm.Dis;
8514	unsigned cbInstr;
8515	int rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbInstr);
8516	if (RT_SUCCESS(rc))
8517	{
8518	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8519	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq /, 1 / u8Level /, 0 / uTagSrc */);
8520	if (RT_SUCCESS(rc))
8521	hmR0SvmAdvanceRip(pVCpu, cbInstr);
8522	}
8523	else
8524	Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8525	return rc;
8526	}
8527
8528	hmR0SvmSetPendingXcptMF(pVCpu);
8529	return VINF_SUCCESS;
8530	}
8531
8532
8533	/**
8534	* \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8535	* \#VMEXIT.
8536	*/
8537	HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8538	{
8539	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8540	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8541	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8542	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8543
8544	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8545	{
8546	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8547	return VINF_EM_RAW_INJECT_TRPM_EVENT;
8548	}
8549
8550	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8551
8552	/*
8553	* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8554	* breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8555	* handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8556	*/
8557	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8558	PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8559	int rc = DBGFTrap01Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx, pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8560	if (rc == VINF_EM_RAW_GUEST_TRAP)
8561	{
8562	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8563	if (CPUMIsHyperDebugStateActive(pVCpu))
8564	CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) \| pVmcb->guest.u64DR6);
8565
8566	/* Reflect the exception back to the guest. */
8567	hmR0SvmSetPendingXcptDB(pVCpu);
8568	rc = VINF_SUCCESS;
8569	}
8570
8571	/*
8572	* Update DR6.
8573	*/
8574	if (CPUMIsHyperDebugStateActive(pVCpu))
8575	{
8576	Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8577	pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8578	pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8579	}
8580	else
8581	{
8582	AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8583	Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8584	}
8585
8586	return rc;
8587	}
8588
8589
8590	/**
8591	* \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8592	* Conditional \#VMEXIT.
8593	*/
8594	HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8595	{
8596	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8597	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8598	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8599
8600	SVMEVENT Event;
8601	Event.u = 0;
8602	Event.n.u1Valid = 1;
8603	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8604	Event.n.u8Vector = X86_XCPT_AC;
8605	Event.n.u1ErrorCodeValid = 1;
8606	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8607	return VINF_SUCCESS;
8608	}
8609
8610
8611	/**
8612	* \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8613	* Conditional \#VMEXIT.
8614	*/
8615	HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8616	{
8617	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8618	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8619	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8620	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8621
8622	VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
8623	if (rc == VINF_EM_RAW_GUEST_TRAP)
8624	{
8625	SVMEVENT Event;
8626	Event.u = 0;
8627	Event.n.u1Valid = 1;
8628	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8629	Event.n.u8Vector = X86_XCPT_BP;
8630	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8631	rc = VINF_SUCCESS;
8632	}
8633
8634	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_DBG_BREAKPOINT);
8635	return rc;
8636	}
8637
8638
8639	/**
8640	* Hacks its way around the lovely mesa driver's backdoor accesses.
8641	*
8642	* @sa hmR0VmxHandleMesaDrvGp
8643	*/
8644	static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8645	{
8646	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_GPRS_MASK);
8647	Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8648	pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8649	RT_NOREF(pCtx, pVmcb);
8650
8651	/* For now we'll just skip the instruction. */
8652	hmR0SvmAdvanceRip(pVCpu, 1);
8653	return VINF_SUCCESS;
8654	}
8655
8656
8657	/**
8658	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8659	* backdoor logging w/o checking what it is running inside.
8660	*
8661	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8662	* backdoor port and magic numbers loaded in registers.
8663	*
8664	* @returns true if it is, false if it isn't.
8665	* @sa hmR0VmxIsMesaDrvGp
8666	*/
8667	DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8668	{
8669	/* Check magic and port. */
8670	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
8671	/Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));/
8672	if (pCtx->dx != UINT32_C(0x5658))
8673	return false;
8674	if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8675	return false;
8676
8677	/* Check that it is #GP(0). */
8678	if (pVmcb->ctrl.u64ExitInfo1 != 0)
8679	return false;
8680
8681	/* Flat ring-3 CS. */
8682	/Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));/
8683	if (pVmcb->guest.u8CPL != 3)
8684	return false;
8685	if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8686	return false;
8687
8688	/* 0xed: IN eAX,dx */
8689	if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8690	{
8691	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_GPRS_MASK
8692	\| CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_EFER);
8693	uint8_t abInstr[1];
8694	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8695	/Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); /
8696	if (RT_FAILURE(rc))
8697	return false;
8698	if (abInstr[0] != 0xed)
8699	return false;
8700	}
8701	else
8702	{
8703	/Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));/
8704	if (pVmcb->ctrl.abInstr[0] != 0xed)
8705	return false;
8706	}
8707	return true;
8708	}
8709
8710
8711	/**
8712	* \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8713	* Conditional \#VMEXIT.
8714	*/
8715	HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8716	{
8717	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8718	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8720
8721	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8722	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8723
8724	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8725	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8726	\|\| !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8727	{
8728	SVMEVENT Event;
8729	Event.u = 0;
8730	Event.n.u1Valid = 1;
8731	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8732	Event.n.u8Vector = X86_XCPT_GP;
8733	Event.n.u1ErrorCodeValid = 1;
8734	Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8735	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8736	return VINF_SUCCESS;
8737	}
8738	return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8739	}
8740
8741
8742	/**
8743	* \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8744	*/
8745	HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8746	{
8747	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8748	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8749
8750	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8751	uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8752	uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8753	Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8754	Assert(uVector <= X86_XCPT_LAST);
8755	Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8756
8757	SVMEVENT Event;
8758	Event.u = 0;
8759	Event.n.u1Valid = 1;
8760	Event.n.u3Type = SVM_EVENT_EXCEPTION;
8761	Event.n.u8Vector = uVector;
8762	switch (uVector)
8763	{
8764	/* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8765	case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8766	case X86_XCPT_DF:
8767	case X86_XCPT_TS:
8768	case X86_XCPT_NP:
8769	case X86_XCPT_SS:
8770	case X86_XCPT_GP:
8771	case X86_XCPT_AC:
8772	{
8773	Event.n.u1ErrorCodeValid = 1;
8774	Event.n.u32ErrorCode = uErrCode;
8775	break;
8776	}
8777	}
8778
8779	#ifdef VBOX_WITH_STATISTICS
8780	switch (uVector)
8781	{
8782	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8783	case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8784	case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8785	case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8786	case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8787	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8788	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8789	case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8790	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8791	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8792	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8793	case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8794	case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8795	case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8796	case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8797	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8798	default:
8799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8800	break;
8801	}
8802	#endif
8803
8804	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8805	return VINF_SUCCESS;
8806	}
8807
8808
8809	/**
8810	* \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8811	*/
8812	HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8813	{
8814	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8815	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8816
8817	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8818	SVMEVENT Event;
8819	Event.u = 0;
8820	Event.n.u1Valid = 1;
8821	Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8822	Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8823	Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8824	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8825	return VINF_SUCCESS;
8826	}
8827
8828
8829	/**
8830	* Generic exit handler that interprets the current instruction
8831	*
8832	* Useful exit that only gets triggered by dtrace and the debugger. Caller does
8833	* the exit logging, and this function does the rest.
8834	*/
8835	static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8836	uint64_t fExtraImport, uint64_t fHmChanged)
8837	{
8838	#if 1
8839	RT_NOREF(pSvmTransient);
8840	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fExtraImport);
8841	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8842	if (rcStrict == VINF_SUCCESS)
8843	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RIP);
8844	else
8845	{
8846	Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8847	if (rcStrict == VINF_IEM_RAISED_XCPT)
8848	{
8849	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK \| fHmChanged);
8850	rcStrict = VINF_SUCCESS;
8851	}
8852	else
8853	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8854	}
8855	return rcStrict;
8856	#else
8857	RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8858	return VINF_EM_RAW_EMULATE_INSTR;
8859	#endif
8860	}
8861
8862
8863	/**
8864	* \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8865	*/
8866	HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8867	{
8868	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8869	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8870	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8871	}
8872
8873
8874	/**
8875	* \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8876	*/
8877	HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8878	{
8879	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8880
8881	/* Workaround for lack of TLB flushing in OS/2 when returning to protected
8882	mode after a real mode call (like a BIOS call). See ticketref:20625
8883	comment 14. */
8884	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8885	if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8886	{
8887	Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8888	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8889	}
8890	else
8891	Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8892
8893	return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8894	}
8895
8896
8897	#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8898	/**
8899	* \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8900	*/
8901	HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8902	{
8903	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8904
8905	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8906	Assert(pVmcb);
8907	Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8908
8909	VBOXSTRICTRC rcStrict;
8910	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8911	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8912	if (fSupportsNextRipSave)
8913	{
8914	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8915	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8916	rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8917	}
8918	else
8919	{
8920	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8921	rcStrict = IEMExecOne(pVCpu);
8922	}
8923
8924	if (rcStrict == VINF_SUCCESS)
8925	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8926	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8927	{
8928	rcStrict = VINF_SUCCESS;
8929	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8930	}
8931	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8932	return rcStrict;
8933	}
8934
8935
8936	/**
8937	* \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8938	*/
8939	HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8940	{
8941	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8942
8943	/*
8944	* When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8945	* we only intercept STGI when events are pending for GIF to become 1.
8946	*/
8947	PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8948	if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8949	hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8950
8951	VBOXSTRICTRC rcStrict;
8952	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8953	uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8954	if (fSupportsNextRipSave)
8955	{
8956	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8957	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8958	rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8959	}
8960	else
8961	{
8962	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
8963	rcStrict = IEMExecOne(pVCpu);
8964	}
8965
8966	if (rcStrict == VINF_SUCCESS)
8967	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8968	else if (rcStrict == VINF_IEM_RAISED_XCPT)
8969	{
8970	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8971	rcStrict = VINF_SUCCESS;
8972	}
8973	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8974	return rcStrict;
8975	}
8976
8977
8978	/**
8979	* \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8980	*/
8981	HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8982	{
8983	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8984
8985	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8986	Assert(pVmcb);
8987	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8988
8989	VBOXSTRICTRC rcStrict;
8990	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8991	uint64_t const fImport = CPUMCTX_EXTRN_FS \| CPUMCTX_EXTRN_GS \| CPUMCTX_EXTRN_KERNEL_GS_BASE
8992	\| CPUMCTX_EXTRN_TR \| CPUMCTX_EXTRN_LDTR \| CPUMCTX_EXTRN_SYSCALL_MSRS
8993	\| CPUMCTX_EXTRN_SYSENTER_MSRS;
8994	if (fSupportsNextRipSave)
8995	{
8996	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| fImport);
8997	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8998	rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
8999	}
9000	else
9001	{
9002	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK \| fImport);
9003	rcStrict = IEMExecOne(pVCpu);
9004	}
9005
9006	if (rcStrict == VINF_SUCCESS)
9007	{
9008	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS \| HM_CHANGED_GUEST_GS
9009	\| HM_CHANGED_GUEST_TR \| HM_CHANGED_GUEST_LDTR
9010	\| HM_CHANGED_GUEST_KERNEL_GS_BASE \| HM_CHANGED_GUEST_SYSCALL_MSRS
9011	\| HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
9012	}
9013	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9014	{
9015	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9016	rcStrict = VINF_SUCCESS;
9017	}
9018	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9019	return rcStrict;
9020	}
9021
9022
9023	/**
9024	* \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9025	*/
9026	HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9027	{
9028	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9029
9030	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9031	Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9032
9033	VBOXSTRICTRC rcStrict;
9034	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9035	if (fSupportsNextRipSave)
9036	{
9037	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9038	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9039	rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9040	}
9041	else
9042	{
9043	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9044	rcStrict = IEMExecOne(pVCpu);
9045	}
9046
9047	if (rcStrict == VINF_IEM_RAISED_XCPT)
9048	{
9049	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9050	rcStrict = VINF_SUCCESS;
9051	}
9052	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9053	return rcStrict;
9054	}
9055
9056
9057	/**
9058	* \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9059	*/
9060	HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9061	{
9062	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9063
9064	VBOXSTRICTRC rcStrict;
9065	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9066	if (fSupportsNextRipSave)
9067	{
9068	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9069	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9070	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9071	rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9072	}
9073	else
9074	{
9075	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9076	rcStrict = IEMExecOne(pVCpu);
9077	}
9078
9079	if (rcStrict == VINF_IEM_RAISED_XCPT)
9080	{
9081	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9082	rcStrict = VINF_SUCCESS;
9083	}
9084	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9085	return rcStrict;
9086	}
9087
9088
9089	/**
9090	* \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9091	*/
9092	HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9093	{
9094	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9095	/* We shall import the entire state here, just in case we enter and continue execution of
9096	the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9097	could lose lose part of CPU state. */
9098	HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9099
9100	VBOXSTRICTRC rcStrict;
9101	bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9102	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9103	if (fSupportsNextRipSave)
9104	{
9105	PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9106	uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9107	rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9108	}
9109	else
9110	{
9111	/* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
9112	instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9113	instruction itself, see @bugref{7243#c126} */
9114	rcStrict = IEMExecOneBypassEx(pVCpu, NULL /* pcbWritten */);
9115	}
9116	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9117
9118	if (rcStrict == VINF_SUCCESS)
9119	{
9120	rcStrict = VINF_SVM_VMRUN;
9121	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9122	}
9123	else if (rcStrict == VINF_IEM_RAISED_XCPT)
9124	{
9125	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9126	rcStrict = VINF_SUCCESS;
9127	}
9128	HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9129	return rcStrict;
9130	}
9131
9132
9133	/**
9134	* Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9135	* Unconditional \#VMEXIT.
9136	*/
9137	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9138	{
9139	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9140	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9141
9142	if (pVCpu->hm.s.Event.fPending)
9143	{
9144	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9145	return VINF_EM_RAW_INJECT_TRPM_EVENT;
9146	}
9147
9148	hmR0SvmSetPendingXcptDB(pVCpu);
9149	return VINF_SUCCESS;
9150	}
9151
9152
9153	/**
9154	* Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9155	* Conditional \#VMEXIT.
9156	*/
9157	HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9158	{
9159	HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9160	HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9161
9162	SVMEVENT Event;
9163	Event.u = 0;
9164	Event.n.u1Valid = 1;
9165	Event.n.u3Type = SVM_EVENT_EXCEPTION;
9166	Event.n.u8Vector = X86_XCPT_BP;
9167	hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9168	return VINF_SUCCESS;
9169	}
9170	#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9171
9172	/** @} */
9173

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 99725

Download in other formats: