VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 72462

Last change on this file since 72462 was 72462, checked in by vboxsync, 7 years ago

EM,IEM,HM: Consolidated VMMHypercallsDisable/VMMHypercallsEnable into EMSetHypercallInstructionsEnabled and made the information available thru EMAreHypercallInstructionsEnabled(). Adjusted the vmmcall implementation so it works without HM. bugref:9044

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 305.7 KB
Line 
1/* $Id: HMSVMR0.cpp 72462 2018-06-06 14:24:04Z vboxsync $ */
2/** @file
3 * HM SVM (AMD-V) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2013-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/tm.h>
32#include <VBox/vmm/gim.h>
33#include <VBox/vmm/apic.h>
34#include "HMInternal.h"
35#include <VBox/vmm/vm.h>
36#include "HMSVMR0.h"
37#include "dtrace/VBoxVMM.h"
38
39#define HMSVM_USE_IEM_EVENT_REFLECTION
40#ifdef DEBUG_ramshankar
41# define HMSVM_SYNC_FULL_GUEST_STATE
42# define HMSVM_SYNC_FULL_NESTED_GUEST_STATE
43# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
44# define HMSVM_ALWAYS_TRAP_PF
45# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
46#endif
47
48
49/*********************************************************************************************************************************
50* Defined Constants And Macros *
51*********************************************************************************************************************************/
52#ifdef VBOX_WITH_STATISTICS
53# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
54 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
55 if ((u64ExitCode) == SVM_EXIT_NPF) \
56 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
57 else \
58 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
59 } while (0)
60
61# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
62# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
63 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
64 if ((u64ExitCode) == SVM_EXIT_NPF) \
65 STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
66 else \
67 STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
68 } while (0)
69# endif
70#else
71# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
72# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
73# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
74# endif
75#endif /* !VBOX_WITH_STATISTICS */
76
77
78/** If we decide to use a function table approach this can be useful to
79 * switch to a "static DECLCALLBACK(int)". */
80#define HMSVM_EXIT_DECL static int
81
82/** Macro for checking and returning from the using function for
83 * \#VMEXIT intercepts that maybe caused during delivering of another
84 * event in the guest. */
85#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
86# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
87 do \
88 { \
89 int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
90 if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
91 else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
92 else if ( rc == VINF_EM_RESET \
93 && CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
94 return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_SHUTDOWN, 0, 0)); \
95 else \
96 return rc; \
97 } while (0)
98#else
99# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
100 do \
101 { \
102 int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
103 if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
104 else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
105 else \
106 return rc; \
107 } while (0)
108#endif
109
110/**
111 * Updates interrupt shadow for the current RIP.
112 */
113#define HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx) \
114 do { \
115 /* Update interrupt shadow. */ \
116 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS) \
117 && pCtx->rip != EMGetInhibitInterruptsPC(pVCpu)) \
118 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); \
119 } while (0)
120
121/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
122 * instruction that exited. */
123#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
124 do { \
125 if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
126 (a_rc) = VINF_EM_DBG_STEPPED; \
127 } while (0)
128
129/** Assert that preemption is disabled or covered by thread-context hooks. */
130#define HMSVM_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
131 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
132
133/** Assert that we haven't migrated CPUs when thread-context hooks are not
134 * used. */
135#define HMSVM_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
136 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
137 ("Illegal migration! Entered on CPU %u Current %u\n", \
138 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()));
139
140/** Assert that we're not executing a nested-guest. */
141#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
142# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
143#else
144# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
145#endif
146
147/** Assert that we're executing a nested-guest. */
148#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
149# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
150#else
151# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
152#endif
153
154/** Validate segment descriptor granularity bit. */
155#ifdef VBOX_STRICT
156# define HMSVM_ASSERT_SEG_GRANULARITY(reg) \
157 AssertMsg( !pMixedCtx->reg.Attr.n.u1Present \
158 || ( pMixedCtx->reg.Attr.n.u1Granularity \
159 ? (pMixedCtx->reg.u32Limit & 0xfff) == 0xfff \
160 : pMixedCtx->reg.u32Limit <= UINT32_C(0xfffff)), \
161 ("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", pMixedCtx->reg.u32Limit, \
162 pMixedCtx->reg.Attr.u, pMixedCtx->reg.u64Base))
163#else
164# define HMSVM_ASSERT_SEG_GRANULARITY(reg) do { } while (0)
165#endif
166
167/**
168 * Exception bitmap mask for all contributory exceptions.
169 *
170 * Page fault is deliberately excluded here as it's conditional as to whether
171 * it's contributory or benign. Page faults are handled separately.
172 */
173#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
174 | RT_BIT(X86_XCPT_DE))
175
176/**
177 * Mandatory/unconditional guest control intercepts.
178 *
179 * SMIs can and do happen in normal operation. We need not intercept them
180 * while executing the guest (or nested-guest).
181 */
182#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
183 | SVM_CTRL_INTERCEPT_NMI \
184 | SVM_CTRL_INTERCEPT_INIT \
185 | SVM_CTRL_INTERCEPT_RDPMC \
186 | SVM_CTRL_INTERCEPT_CPUID \
187 | SVM_CTRL_INTERCEPT_RSM \
188 | SVM_CTRL_INTERCEPT_HLT \
189 | SVM_CTRL_INTERCEPT_IOIO_PROT \
190 | SVM_CTRL_INTERCEPT_MSR_PROT \
191 | SVM_CTRL_INTERCEPT_INVLPGA \
192 | SVM_CTRL_INTERCEPT_SHUTDOWN \
193 | SVM_CTRL_INTERCEPT_FERR_FREEZE \
194 | SVM_CTRL_INTERCEPT_VMRUN \
195 | SVM_CTRL_INTERCEPT_SKINIT \
196 | SVM_CTRL_INTERCEPT_WBINVD \
197 | SVM_CTRL_INTERCEPT_MONITOR \
198 | SVM_CTRL_INTERCEPT_MWAIT \
199 | SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
200 | SVM_CTRL_INTERCEPT_XSETBV)
201
202/** @name VMCB Clean Bits.
203 *
204 * These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
205 * AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
206 * memory.
207 *
208 * @{ */
209/** All intercepts vectors, TSC offset, PAUSE filter counter. */
210#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
211/** I/O permission bitmap, MSR permission bitmap. */
212#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
213/** ASID. */
214#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
215/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
216V_INTR_VECTOR. */
217#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
218/** Nested Paging: Nested CR3 (nCR3), PAT. */
219#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
220/** Control registers (CR0, CR3, CR4, EFER). */
221#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
222/** Debug registers (DR6, DR7). */
223#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
224/** GDT, IDT limit and base. */
225#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
226/** Segment register: CS, SS, DS, ES limit and base. */
227#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
228/** CR2.*/
229#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
230/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
231#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
232/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
233PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
234#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
235/** Mask of all valid VMCB Clean bits. */
236#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
237 | HMSVM_VMCB_CLEAN_IOPM_MSRPM \
238 | HMSVM_VMCB_CLEAN_ASID \
239 | HMSVM_VMCB_CLEAN_INT_CTRL \
240 | HMSVM_VMCB_CLEAN_NP \
241 | HMSVM_VMCB_CLEAN_CRX_EFER \
242 | HMSVM_VMCB_CLEAN_DRX \
243 | HMSVM_VMCB_CLEAN_DT \
244 | HMSVM_VMCB_CLEAN_SEG \
245 | HMSVM_VMCB_CLEAN_CR2 \
246 | HMSVM_VMCB_CLEAN_LBR \
247 | HMSVM_VMCB_CLEAN_AVIC)
248/** @} */
249
250/** @name SVM transient.
251 *
252 * A state structure for holding miscellaneous information across AMD-V
253 * VMRUN/\#VMEXIT operation, restored after the transition.
254 *
255 * @{ */
256typedef struct SVMTRANSIENT
257{
258 /** The host's rflags/eflags. */
259 RTCCUINTREG fEFlags;
260#if HC_ARCH_BITS == 32
261 uint32_t u32Alignment0;
262#endif
263
264 /** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
265 uint64_t u64ExitCode;
266 /** The guest's TPR value used for TPR shadowing. */
267 uint8_t u8GuestTpr;
268 /** Alignment. */
269 uint8_t abAlignment0[7];
270
271 /** Pointer to the currently executing VMCB. */
272 PSVMVMCB pVmcb;
273 /** Whether we are currently executing a nested-guest. */
274 bool fIsNestedGuest;
275
276 /** Whether the guest debug state was active at the time of \#VMEXIT. */
277 bool fWasGuestDebugStateActive;
278 /** Whether the hyper debug state was active at the time of \#VMEXIT. */
279 bool fWasHyperDebugStateActive;
280 /** Whether the TSC offset mode needs to be updated. */
281 bool fUpdateTscOffsetting;
282 /** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
283 bool fRestoreTscAuxMsr;
284 /** Whether the \#VMEXIT was caused by a page-fault during delivery of a
285 * contributary exception or a page-fault. */
286 bool fVectoringDoublePF;
287 /** Whether the \#VMEXIT was caused by a page-fault during delivery of an
288 * external interrupt or NMI. */
289 bool fVectoringPF;
290} SVMTRANSIENT, *PSVMTRANSIENT;
291AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
292AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
293/** @} */
294
295/**
296 * MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
297 */
298typedef enum SVMMSREXITREAD
299{
300 /** Reading this MSR causes a \#VMEXIT. */
301 SVMMSREXIT_INTERCEPT_READ = 0xb,
302 /** Reading this MSR does not cause a \#VMEXIT. */
303 SVMMSREXIT_PASSTHRU_READ
304} SVMMSREXITREAD;
305
306/**
307 * MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
308 */
309typedef enum SVMMSREXITWRITE
310{
311 /** Writing to this MSR causes a \#VMEXIT. */
312 SVMMSREXIT_INTERCEPT_WRITE = 0xd,
313 /** Writing to this MSR does not cause a \#VMEXIT. */
314 SVMMSREXIT_PASSTHRU_WRITE
315} SVMMSREXITWRITE;
316
317/**
318 * SVM \#VMEXIT handler.
319 *
320 * @returns VBox status code.
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context.
323 * @param pSvmTransient Pointer to the SVM-transient structure.
324 */
325typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
326
327
328/*********************************************************************************************************************************
329* Internal Functions *
330*********************************************************************************************************************************/
331static void hmR0SvmSetMsrPermission(PCPUMCTX pCtx, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
332 SVMMSREXITWRITE enmWrite);
333static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
334static void hmR0SvmLeave(PVMCPU pVCpu);
335
336/** @name \#VMEXIT handlers.
337 * @{
338 */
339static FNSVMEXITHANDLER hmR0SvmExitIntr;
340static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
341static FNSVMEXITHANDLER hmR0SvmExitInvd;
342static FNSVMEXITHANDLER hmR0SvmExitCpuid;
343static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
344static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
345static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
346static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
347static FNSVMEXITHANDLER hmR0SvmExitHlt;
348static FNSVMEXITHANDLER hmR0SvmExitMonitor;
349static FNSVMEXITHANDLER hmR0SvmExitMwait;
350static FNSVMEXITHANDLER hmR0SvmExitShutdown;
351static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
352static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
353static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
354static FNSVMEXITHANDLER hmR0SvmExitMsr;
355static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
356static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
357static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
358static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
359static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
360static FNSVMEXITHANDLER hmR0SvmExitVIntr;
361static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
362static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
363static FNSVMEXITHANDLER hmR0SvmExitPause;
364static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
365static FNSVMEXITHANDLER hmR0SvmExitIret;
366static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
367static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
368static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
369static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
370static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
371static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
372#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(VBOX_WITH_NESTED_HWVIRT_SVM)
373static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
374#endif
375#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
376static FNSVMEXITHANDLER hmR0SvmExitClgi;
377static FNSVMEXITHANDLER hmR0SvmExitStgi;
378static FNSVMEXITHANDLER hmR0SvmExitVmload;
379static FNSVMEXITHANDLER hmR0SvmExitVmsave;
380static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
381static FNSVMEXITHANDLER hmR0SvmExitVmrun;
382static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
383static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
384#endif
385/** @} */
386
387static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient);
388#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
389static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
390#endif
391
392
393/*********************************************************************************************************************************
394* Global Variables *
395*********************************************************************************************************************************/
396/** Ring-0 memory object for the IO bitmap. */
397static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
398/** Physical address of the IO bitmap. */
399static RTHCPHYS g_HCPhysIOBitmap;
400/** Pointer to the IO bitmap. */
401static R0PTRTYPE(void *) g_pvIOBitmap;
402
403#ifdef VBOX_STRICT
404# define HMSVM_LOG_CS RT_BIT_32(0)
405# define HMSVM_LOG_SS RT_BIT_32(1)
406# define HMSVM_LOG_FS RT_BIT_32(2)
407# define HMSVM_LOG_GS RT_BIT_32(3)
408# define HMSVM_LOG_LBR RT_BIT_32(4)
409# define HMSVM_LOG_ALL ( HMSVM_LOG_CS \
410 | HMSVM_LOG_SS \
411 | HMSVM_LOG_FS \
412 | HMSVM_LOG_GS \
413 | HMSVM_LOG_LBR)
414
415/**
416 * Dumps virtual CPU state and additional info. to the logger for diagnostics.
417 *
418 * @param pVCpu The cross context virtual CPU structure.
419 * @param pVmcb Pointer to the VM control block.
420 * @param pCtx Pointer to the guest-CPU context.
421 * @param pszPrefix Log prefix.
422 * @param fFlags Log flags, see HMSVM_LOG_XXX.
423 * @param uVerbose The verbosity level, currently unused.
424 */
425static void hmR0SvmLogState(PVMCPU pVCpu, PCSVMVMCB pVmcb, PCPUMCTX pCtx, const char *pszPrefix, uint32_t fFlags,
426 uint8_t uVerbose)
427{
428 RT_NOREF2(pVCpu, uVerbose);
429
430 Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip,
431 pCtx->rflags.u, pCtx->cr0, pCtx->cr3, pCtx->cr4));
432 Log4(("%s: rsp=%#RX64 rbp=%#RX64 rdi=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp, pCtx->rdi));
433 if (fFlags & HMSVM_LOG_CS)
434 {
435 Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
436 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
437 }
438 if (fFlags & HMSVM_LOG_SS)
439 {
440 Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
441 pCtx->ss.u32Limit, pCtx->ss.Attr.u));
442 }
443 if (fFlags & HMSVM_LOG_FS)
444 {
445 Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
446 pCtx->fs.u32Limit, pCtx->fs.Attr.u));
447 }
448 if (fFlags & HMSVM_LOG_GS)
449 {
450 Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
451 pCtx->gs.u32Limit, pCtx->gs.Attr.u));
452 }
453
454 PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
455 if (fFlags & HMSVM_LOG_LBR)
456 {
457 Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
458 pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
459 }
460 NOREF(pVmcbGuest);
461}
462#endif /* VBOX_STRICT */
463
464
465/**
466 * Sets up and activates AMD-V on the current CPU.
467 *
468 * @returns VBox status code.
469 * @param pCpu Pointer to the CPU info struct.
470 * @param pVM The cross context VM structure. Can be
471 * NULL after a resume!
472 * @param pvCpuPage Pointer to the global CPU page.
473 * @param HCPhysCpuPage Physical address of the global CPU page.
474 * @param fEnabledByHost Whether the host OS has already initialized AMD-V.
475 * @param pvArg Unused on AMD-V.
476 */
477VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
478 void *pvArg)
479{
480 Assert(!fEnabledByHost);
481 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
482 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
483 Assert(pvCpuPage); NOREF(pvCpuPage);
484 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
485
486 NOREF(pvArg);
487 NOREF(fEnabledByHost);
488
489 /* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
490 RTCCUINTREG fEFlags = ASMIntDisableFlags();
491
492 /*
493 * We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
494 */
495 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
496 if (u64HostEfer & MSR_K6_EFER_SVME)
497 {
498 /* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
499 if ( pVM
500 && pVM->hm.s.svm.fIgnoreInUseError)
501 {
502 pCpu->fIgnoreAMDVInUseError = true;
503 }
504
505 if (!pCpu->fIgnoreAMDVInUseError)
506 {
507 ASMSetFlags(fEFlags);
508 return VERR_SVM_IN_USE;
509 }
510 }
511
512 /* Turn on AMD-V in the EFER MSR. */
513 ASMWrMsr(MSR_K6_EFER, u64HostEfer | MSR_K6_EFER_SVME);
514
515 /* Write the physical page address where the CPU will store the host state while executing the VM. */
516 ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
517
518 /* Restore interrupts. */
519 ASMSetFlags(fEFlags);
520
521 /*
522 * Theoretically, other hypervisors may have used ASIDs, ideally we should flush all non-zero ASIDs
523 * when enabling SVM. AMD doesn't have an SVM instruction to flush all ASIDs (flushing is done
524 * upon VMRUN). Therefore, flag that we need to flush the TLB entirely with before executing any
525 * guest code.
526 */
527 pCpu->fFlushAsidBeforeUse = true;
528
529 /*
530 * Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
531 */
532 ++pCpu->cTlbFlushes;
533
534 return VINF_SUCCESS;
535}
536
537
538/**
539 * Deactivates AMD-V on the current CPU.
540 *
541 * @returns VBox status code.
542 * @param pCpu Pointer to the CPU info struct.
543 * @param pvCpuPage Pointer to the global CPU page.
544 * @param HCPhysCpuPage Physical address of the global CPU page.
545 */
546VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
547{
548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
549 AssertReturn( HCPhysCpuPage
550 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
551 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
552 NOREF(pCpu);
553
554 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
555 RTCCUINTREG fEFlags = ASMIntDisableFlags();
556
557 /* Turn off AMD-V in the EFER MSR. */
558 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
559 ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
560
561 /* Invalidate host state physical address. */
562 ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
563
564 /* Restore interrupts. */
565 ASMSetFlags(fEFlags);
566
567 return VINF_SUCCESS;
568}
569
570
571/**
572 * Does global AMD-V initialization (called during module initialization).
573 *
574 * @returns VBox status code.
575 */
576VMMR0DECL(int) SVMR0GlobalInit(void)
577{
578 /*
579 * Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
580 * intercept all IO accesses, it's done once globally here instead of per-VM.
581 */
582 Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
583 int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
584 if (RT_FAILURE(rc))
585 return rc;
586
587 g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
588 g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
589
590 /* Set all bits to intercept all IO accesses. */
591 ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
592
593 return VINF_SUCCESS;
594}
595
596
597/**
598 * Does global AMD-V termination (called during module termination).
599 */
600VMMR0DECL(void) SVMR0GlobalTerm(void)
601{
602 if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
603 {
604 RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
605 g_pvIOBitmap = NULL;
606 g_HCPhysIOBitmap = 0;
607 g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
608 }
609}
610
611
612/**
613 * Frees any allocated per-VCPU structures for a VM.
614 *
615 * @param pVM The cross context VM structure.
616 */
617DECLINLINE(void) hmR0SvmFreeStructs(PVM pVM)
618{
619 for (uint32_t i = 0; i < pVM->cCpus; i++)
620 {
621 PVMCPU pVCpu = &pVM->aCpus[i];
622 AssertPtr(pVCpu);
623
624 if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
625 {
626 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
627 pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
628 pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
629 }
630
631 if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
632 {
633 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
634 pVCpu->hm.s.svm.pVmcb = NULL;
635 pVCpu->hm.s.svm.HCPhysVmcb = 0;
636 pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
637 }
638
639 if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
640 {
641 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
642 pVCpu->hm.s.svm.pvMsrBitmap = NULL;
643 pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
644 pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
645 }
646 }
647}
648
649
650/**
651 * Does per-VM AMD-V initialization.
652 *
653 * @returns VBox status code.
654 * @param pVM The cross context VM structure.
655 */
656VMMR0DECL(int) SVMR0InitVM(PVM pVM)
657{
658 int rc = VERR_INTERNAL_ERROR_5;
659
660 /*
661 * Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
662 */
663 uint32_t u32Family;
664 uint32_t u32Model;
665 uint32_t u32Stepping;
666 if (HMAmdIsSubjectToErratum170(&u32Family, &u32Model, &u32Stepping))
667 {
668 Log4(("SVMR0InitVM: AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
669 pVM->hm.s.svm.fAlwaysFlushTLB = true;
670 }
671
672 /*
673 * Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
674 */
675 for (VMCPUID i = 0; i < pVM->cCpus; i++)
676 {
677 PVMCPU pVCpu = &pVM->aCpus[i];
678 pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
679 pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
680 pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
681 }
682
683 for (VMCPUID i = 0; i < pVM->cCpus; i++)
684 {
685 PVMCPU pVCpu = &pVM->aCpus[i];
686
687 /*
688 * Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
689 * FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
690 */
691 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
692 if (RT_FAILURE(rc))
693 goto failure_cleanup;
694
695 void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
696 pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
697 Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
698 ASMMemZeroPage(pvVmcbHost);
699
700 /*
701 * Allocate one page for the guest-state VMCB.
702 */
703 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
704 if (RT_FAILURE(rc))
705 goto failure_cleanup;
706
707 pVCpu->hm.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
708 pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
709 Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
710 ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
711
712 /*
713 * Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
714 * SVM to not require one.
715 */
716 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
717 false /* fExecutable */);
718 if (RT_FAILURE(rc))
719 goto failure_cleanup;
720
721 pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
722 pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
723 /* Set all bits to intercept all MSR accesses (changed later on). */
724 ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
725 }
726
727 return VINF_SUCCESS;
728
729failure_cleanup:
730 hmR0SvmFreeStructs(pVM);
731 return rc;
732}
733
734
735/**
736 * Does per-VM AMD-V termination.
737 *
738 * @returns VBox status code.
739 * @param pVM The cross context VM structure.
740 */
741VMMR0DECL(int) SVMR0TermVM(PVM pVM)
742{
743 hmR0SvmFreeStructs(pVM);
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Returns whether the VMCB Clean Bits feature is supported.
750 *
751 * @return @c true if supported, @c false otherwise.
752 * @param pVCpu The cross context virtual CPU structure.
753 * @param pCtx Pointer to the guest-CPU context.
754 */
755DECLINLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPU pVCpu, PCPUMCTX pCtx)
756{
757 PVM pVM = pVCpu->CTX_SUFF(pVM);
758#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
759 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
760 {
761 return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN)
762 && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
763 }
764#else
765 RT_NOREF(pCtx);
766#endif
767 return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
768}
769
770
771/**
772 * Returns whether the decode assists feature is supported.
773 *
774 * @return @c true if supported, @c false otherwise.
775 * @param pVCpu The cross context virtual CPU structure.
776 * @param pCtx Pointer to the guest-CPU context.
777 */
778DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPU pVCpu, PCPUMCTX pCtx)
779{
780 PVM pVM = pVCpu->CTX_SUFF(pVM);
781#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
782 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
783 {
784 return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
785 && pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
786 }
787#else
788 RT_NOREF(pCtx);
789#endif
790 return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
791}
792
793
794/**
795 * Returns whether the NRIP_SAVE feature is supported.
796 *
797 * @return @c true if supported, @c false otherwise.
798 * @param pVCpu The cross context virtual CPU structure.
799 * @param pCtx Pointer to the guest-CPU context.
800 */
801DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPU pVCpu, PCPUMCTX pCtx)
802{
803 PVM pVM = pVCpu->CTX_SUFF(pVM);
804#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
805 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
806 {
807 return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
808 && pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
809 }
810#else
811 RT_NOREF(pCtx);
812#endif
813 return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
814}
815
816
817/**
818 * Sets the permission bits for the specified MSR in the MSRPM bitmap.
819 *
820 * @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
821 * @param pbMsrBitmap Pointer to the MSR bitmap.
822 * @param idMsr The MSR for which the permissions are being set.
823 * @param enmRead MSR read permissions.
824 * @param enmWrite MSR write permissions.
825 *
826 * @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
827 * caller needs to take care of this.
828 */
829static void hmR0SvmSetMsrPermission(PCPUMCTX pCtx, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
830 SVMMSREXITWRITE enmWrite)
831{
832 bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
833 uint16_t offMsrpm;
834 uint8_t uMsrpmBit;
835 int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
836 AssertRC(rc);
837
838 Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
839 Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
840
841 pbMsrBitmap += offMsrpm;
842 if (enmRead == SVMMSREXIT_INTERCEPT_READ)
843 *pbMsrBitmap |= RT_BIT(uMsrpmBit);
844 else
845 {
846 if (!fInNestedGuestMode)
847 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
848#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
849 else
850 {
851 /* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
852 uint8_t const *pbNstGstMsrBitmap = (uint8_t *)pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
853 pbNstGstMsrBitmap += offMsrpm;
854 if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit)))
855 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
856 else
857 Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
858 }
859#endif
860 }
861
862 if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
863 *pbMsrBitmap |= RT_BIT(uMsrpmBit + 1);
864 else
865 {
866 if (!fInNestedGuestMode)
867 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
868#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
869 else
870 {
871 /* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
872 uint8_t const *pbNstGstMsrBitmap = (uint8_t *)pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
873 pbNstGstMsrBitmap += offMsrpm;
874 if (!(*pbNstGstMsrBitmap & RT_BIT(uMsrpmBit + 1)))
875 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
876 else
877 Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
878 }
879#endif
880 }
881}
882
883
884/**
885 * Sets up AMD-V for the specified VM.
886 * This function is only called once per-VM during initalization.
887 *
888 * @returns VBox status code.
889 * @param pVM The cross context VM structure.
890 */
891VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
892{
893 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
894 AssertReturn(pVM, VERR_INVALID_PARAMETER);
895 Assert(pVM->hm.s.svm.fSupported);
896
897 bool const fPauseFilter = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
898 bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
899 bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
900
901 bool const fLbrVirt = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
902 bool const fUseLbrVirt = fLbrVirt; /** @todo CFGM, IEM implementation etc. */
903
904#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
905 bool const fVirtVmsaveVmload = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
906 bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && pVM->hm.s.fNestedPaging;
907
908 bool const fVGif = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
909 bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
910#endif
911
912 PVMCPU pVCpu = &pVM->aCpus[0];
913 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
914 AssertMsgReturn(pVmcb, ("Invalid pVmcb for vcpu[0]\n"), VERR_SVM_INVALID_PVMCB);
915 PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
916
917 /* Always trap #AC for reasons of security. */
918 pVmcbCtrl->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_AC);
919
920 /* Always trap #DB for reasons of security. */
921 pVmcbCtrl->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_DB);
922
923 /* Trap exceptions unconditionally (debug purposes). */
924#ifdef HMSVM_ALWAYS_TRAP_PF
925 pVmcbCtrl->u32InterceptXcpt |= RT_BIT(X86_XCPT_PF);
926#endif
927#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
928 /* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
929 pVmcbCtrl->u32InterceptXcpt |= 0
930 | RT_BIT(X86_XCPT_BP)
931 | RT_BIT(X86_XCPT_DE)
932 | RT_BIT(X86_XCPT_NM)
933 | RT_BIT(X86_XCPT_UD)
934 | RT_BIT(X86_XCPT_NP)
935 | RT_BIT(X86_XCPT_SS)
936 | RT_BIT(X86_XCPT_GP)
937 | RT_BIT(X86_XCPT_PF)
938 | RT_BIT(X86_XCPT_MF)
939 ;
940#endif
941
942 /* Apply the exceptions intercepts needed by the GIM provider. */
943 if (pVCpu->hm.s.fGIMTrapXcptUD)
944 pVmcbCtrl->u32InterceptXcpt |= RT_BIT(X86_XCPT_UD);
945
946 /* Set up unconditional intercepts and conditions. */
947 pVmcbCtrl->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
948 | SVM_CTRL_INTERCEPT_VMMCALL;
949
950#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
951 pVmcbCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_TASK_SWITCH;
952#endif
953
954#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
955 /* Virtualized VMSAVE/VMLOAD. */
956 pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload = fUseVirtVmsaveVmload;
957 if (!fUseVirtVmsaveVmload)
958 {
959 pVmcbCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
960 | SVM_CTRL_INTERCEPT_VMLOAD;
961 }
962
963 /* Virtual GIF. */
964 pVmcbCtrl->IntCtrl.n.u1VGifEnable = fUseVGif;
965 if (!fUseVGif)
966 {
967 pVmcbCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_CLGI
968 | SVM_CTRL_INTERCEPT_STGI;
969 }
970#endif
971
972 /* CR4 writes must always be intercepted for tracking PGM mode changes. */
973 pVmcbCtrl->u16InterceptWrCRx = RT_BIT(4);
974
975 /* Intercept all DRx reads and writes by default. Changed later on. */
976 pVmcbCtrl->u16InterceptRdDRx = 0xffff;
977 pVmcbCtrl->u16InterceptWrDRx = 0xffff;
978
979 /* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
980 pVmcbCtrl->IntCtrl.n.u1VIntrMasking = 1;
981
982 /* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
983 and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
984 pVmcbCtrl->IntCtrl.n.u1IgnoreTPR = 1;
985
986 /* Set the IO permission bitmap physical addresses. */
987 pVmcbCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
988
989 /* LBR virtualization. */
990 pVmcbCtrl->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
991
992 /* The host ASID MBZ, for the guest start with 1. */
993 pVmcbCtrl->TLBCtrl.n.u32ASID = 1;
994
995 /* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
996 pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging = pVM->hm.s.fNestedPaging;
997
998 /* Without Nested Paging, we need additionally intercepts. */
999 if (!pVM->hm.s.fNestedPaging)
1000 {
1001 /* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1002 pVmcbCtrl->u16InterceptRdCRx |= RT_BIT(3);
1003 pVmcbCtrl->u16InterceptWrCRx |= RT_BIT(3);
1004
1005 /* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1006 pVmcbCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_INVLPG
1007 | SVM_CTRL_INTERCEPT_TASK_SWITCH;
1008
1009 /* Page faults must be intercepted to implement shadow paging. */
1010 pVmcbCtrl->u32InterceptXcpt |= RT_BIT(X86_XCPT_PF);
1011 }
1012
1013 /* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1014 if (fUsePauseFilter)
1015 {
1016 Assert(pVM->hm.s.svm.cPauseFilter > 0);
1017 pVmcbCtrl->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1018 if (fPauseFilterThreshold)
1019 pVmcbCtrl->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1020 pVmcbCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_PAUSE;
1021 }
1022
1023 /*
1024 * Setup the MSR permission bitmap.
1025 * The following MSRs are saved/restored automatically during the world-switch.
1026 * Don't intercept guest read/write accesses to these MSRs.
1027 */
1028 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
1029 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1030 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1031 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1032 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1033 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1034 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1035 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1036 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1037 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1038 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1039 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1040 pVmcbCtrl->u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
1041
1042 /* Initialize the #VMEXIT history array with end-of-array markers (UINT16_MAX). */
1043 Assert(!pVCpu->hm.s.idxExitHistoryFree);
1044 HMCPU_EXIT_HISTORY_RESET(pVCpu);
1045
1046 /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1047 Assert(pVmcbCtrl->u32VmcbCleanBits == 0);
1048
1049 for (VMCPUID i = 1; i < pVM->cCpus; i++)
1050 {
1051 PVMCPU pVCpuCur = &pVM->aCpus[i];
1052 PSVMVMCB pVmcbCur = pVM->aCpus[i].hm.s.svm.pVmcb;
1053 AssertMsgReturn(pVmcbCur, ("Invalid pVmcb for vcpu[%u]\n", i), VERR_SVM_INVALID_PVMCB);
1054 PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1055
1056 /* Copy the VMCB control area. */
1057 memcpy(pVmcbCtrlCur, pVmcbCtrl, sizeof(*pVmcbCtrlCur));
1058
1059 /* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1060 uint8_t *pbMsrBitmapCur = (uint8_t *)pVCpuCur->hm.s.svm.pvMsrBitmap;
1061 memcpy(pbMsrBitmapCur, pbMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1062 pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hm.s.svm.HCPhysMsrBitmap;
1063
1064 /* Initialize the #VMEXIT history array with end-of-array markers (UINT16_MAX). */
1065 Assert(!pVCpuCur->hm.s.idxExitHistoryFree);
1066 HMCPU_EXIT_HISTORY_RESET(pVCpuCur);
1067
1068 /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1069 Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1070
1071 /* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1072 Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu->hm.s.fGIMTrapXcptUD);
1073 }
1074
1075 return VINF_SUCCESS;
1076}
1077
1078
1079/**
1080 * Gets a pointer to the currently active guest (or nested-guest) VMCB.
1081 *
1082 * @returns Pointer to the current context VMCB.
1083 * @param pVCpu The cross context virtual CPU structure.
1084 * @param pCtx Pointer to the guest-CPU context.
1085 */
1086DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
1087{
1088#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1089 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1090 return pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
1091#else
1092 RT_NOREF(pCtx);
1093#endif
1094 return pVCpu->hm.s.svm.pVmcb;
1095}
1096
1097
1098/**
1099 * Gets a pointer to the nested-guest VMCB cache.
1100 *
1101 * @returns Pointer to the nested-guest VMCB cache.
1102 * @param pVCpu The cross context virtual CPU structure.
1103 * @param pCtx Pointer to the guest-CPU context.
1104 */
1105DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPU pVCpu, PCPUMCTX pCtx)
1106{
1107#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1108 Assert(pCtx->hwvirt.svm.fHMCachedVmcb); RT_NOREF(pCtx);
1109 return &pVCpu->hm.s.svm.NstGstVmcbCache;
1110#else
1111 RT_NOREF2(pVCpu, pCtx);
1112 return NULL;
1113#endif
1114}
1115
1116
1117/**
1118 * Invalidates a guest page by guest virtual address.
1119 *
1120 * @returns VBox status code.
1121 * @param pVM The cross context VM structure.
1122 * @param pVCpu The cross context virtual CPU structure.
1123 * @param GCVirt Guest virtual address of the page to invalidate.
1124 */
1125VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1126{
1127 AssertReturn(pVM, VERR_INVALID_PARAMETER);
1128 Assert(pVM->hm.s.svm.fSupported);
1129
1130 bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1131
1132 /* Skip it if a TLB flush is already pending. */
1133 if (!fFlushPending)
1134 {
1135 Log4(("SVMR0InvalidatePage %RGv\n", GCVirt));
1136
1137 PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1138 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
1139 AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1140
1141#if HC_ARCH_BITS == 32
1142 /* If we get a flush in 64-bit guest mode, then force a full TLB flush. INVLPGA takes only 32-bit addresses. */
1143 if (CPUMIsGuestInLongMode(pVCpu))
1144 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1145 else
1146#endif
1147 {
1148 SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1149 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1150 }
1151 }
1152 return VINF_SUCCESS;
1153}
1154
1155
1156/**
1157 * Flushes the appropriate tagged-TLB entries.
1158 *
1159 * @param pVCpu The cross context virtual CPU structure.
1160 * @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
1161 * @param pVmcb Pointer to the VM control block.
1162 * @param pHostCpu Pointer to the HM host-CPU info.
1163 */
1164static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb, PHMGLOBALCPUINFO pHostCpu)
1165{
1166#ifndef VBOX_WITH_NESTED_HWVIRT_SVM
1167 RT_NOREF(pCtx);
1168#endif
1169 PVM pVM = pVCpu->CTX_SUFF(pVM);
1170
1171 /*
1172 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1173 * This can happen both for start & resume due to long jumps back to ring-3.
1174 *
1175 * We also force a TLB flush every time when executing a nested-guest VCPU as there is no correlation
1176 * between it and the physical CPU.
1177 *
1178 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
1179 * so we cannot reuse the ASIDs without flushing.
1180 */
1181 bool fNewAsid = false;
1182 Assert(pHostCpu->idCpu != NIL_RTCPUID);
1183 if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
1184 || pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes
1185#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1186 || CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
1187#endif
1188 )
1189 {
1190 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1191 pVCpu->hm.s.fForceTLBFlush = true;
1192 fNewAsid = true;
1193 }
1194
1195 /* Set TLB flush state as checked until we return from the world switch. */
1196 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1197
1198 /* Check for explicit TLB flushes. */
1199 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1200 {
1201 pVCpu->hm.s.fForceTLBFlush = true;
1202 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1203 }
1204
1205 /*
1206 * If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1207 * This Host CPU requirement takes precedence.
1208 */
1209 if (pVM->hm.s.svm.fAlwaysFlushTLB)
1210 {
1211 pHostCpu->uCurrentAsid = 1;
1212 pVCpu->hm.s.uCurrentAsid = 1;
1213 pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1214 pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
1215 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1216
1217 /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1218 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1219 }
1220 else
1221 {
1222 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1223 if (pVCpu->hm.s.fForceTLBFlush)
1224 {
1225 /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1226 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1227
1228 if (fNewAsid)
1229 {
1230 ++pHostCpu->uCurrentAsid;
1231
1232 bool fHitASIDLimit = false;
1233 if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1234 {
1235 pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1236 pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1237 fHitASIDLimit = true;
1238 }
1239
1240 if ( fHitASIDLimit
1241 || pHostCpu->fFlushAsidBeforeUse)
1242 {
1243 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1244 pHostCpu->fFlushAsidBeforeUse = false;
1245 }
1246
1247 pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1248 pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
1249 pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1250 }
1251 else
1252 {
1253 if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1254 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1255 else
1256 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1257 }
1258
1259 pVCpu->hm.s.fForceTLBFlush = false;
1260 }
1261 }
1262
1263 /* Update VMCB with the ASID. */
1264 if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
1265 {
1266 pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
1267 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1268 }
1269
1270 AssertMsg(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu,
1271 ("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pHostCpu->idCpu));
1272 AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1273 ("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1274 AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1275 ("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1276 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1277 ("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1278
1279#ifdef VBOX_WITH_STATISTICS
1280 if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1281 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1282 else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1283 || pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1284 {
1285 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1286 }
1287 else
1288 {
1289 Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1290 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1291 }
1292#endif
1293}
1294
1295
1296/** @name 64-bit guest on 32-bit host OS helper functions.
1297 *
1298 * The host CPU is still 64-bit capable but the host OS is running in 32-bit
1299 * mode (code segment, paging). These wrappers/helpers perform the necessary
1300 * bits for the 32->64 switcher.
1301 *
1302 * @{ */
1303#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1304/**
1305 * Prepares for and executes VMRUN (64-bit guests on a 32-bit host).
1306 *
1307 * @returns VBox status code.
1308 * @param HCPhysVmcbHost Physical address of host VMCB.
1309 * @param HCPhysVmcb Physical address of the VMCB.
1310 * @param pCtx Pointer to the guest-CPU context.
1311 * @param pVM The cross context VM structure.
1312 * @param pVCpu The cross context virtual CPU structure.
1313 */
1314DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1315{
1316 uint32_t aParam[8];
1317 aParam[0] = RT_LO_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Lo. */
1318 aParam[1] = RT_HI_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Hi. */
1319 aParam[2] = RT_LO_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Lo. */
1320 aParam[3] = RT_HI_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Hi. */
1321 aParam[4] = VM_RC_ADDR(pVM, pVM);
1322 aParam[5] = 0;
1323 aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1324 aParam[7] = 0;
1325
1326 return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1327}
1328
1329
1330/**
1331 * Executes the specified VMRUN handler in 64-bit mode.
1332 *
1333 * @returns VBox status code.
1334 * @param pVM The cross context VM structure.
1335 * @param pVCpu The cross context virtual CPU structure.
1336 * @param pCtx Pointer to the guest-CPU context.
1337 * @param enmOp The operation to perform.
1338 * @param cParams Number of parameters.
1339 * @param paParam Array of 32-bit parameters.
1340 */
1341VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
1342 uint32_t cParams, uint32_t *paParam)
1343{
1344 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
1345 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
1346
1347 NOREF(pCtx);
1348
1349 /* Disable interrupts. */
1350 RTHCUINTREG uOldEFlags = ASMIntDisableFlags();
1351
1352#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1353 RTCPUID idHostCpu = RTMpCpuId();
1354 CPUMR0SetLApic(pVCpu, idHostCpu);
1355#endif
1356
1357 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1358 CPUMSetHyperEIP(pVCpu, enmOp);
1359 for (int i = (int)cParams - 1; i >= 0; i--)
1360 CPUMPushHyper(pVCpu, paParam[i]);
1361
1362 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1363 /* Call the switcher. */
1364 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
1365 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1366
1367 /* Restore interrupts. */
1368 ASMSetFlags(uOldEFlags);
1369 return rc;
1370}
1371
1372#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
1373/** @} */
1374
1375
1376/**
1377 * Sets an exception intercept in the specified VMCB.
1378 *
1379 * @param pVmcb Pointer to the VM control block.
1380 * @param uXcpt The exception (X86_XCPT_*).
1381 */
1382DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1383{
1384 if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1385 {
1386 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(uXcpt);
1387 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1388 }
1389}
1390
1391
1392/**
1393 * Clears an exception intercept in the specified VMCB.
1394 *
1395 * @param pVCpu The cross context virtual CPU structure.
1396 * @param pCtx Pointer to the guest-CPU context.
1397 * @param pVmcb Pointer to the VM control block.
1398 * @param uXcpt The exception (X86_XCPT_*).
1399 *
1400 * @remarks This takes into account if we're executing a nested-guest and only
1401 * removes the exception intercept if both the guest -and- nested-guest
1402 * are not intercepting it.
1403 */
1404DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb, uint8_t uXcpt)
1405{
1406 Assert(uXcpt != X86_XCPT_DB);
1407 Assert(uXcpt != X86_XCPT_AC);
1408#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1409 if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1410 {
1411 bool fRemove = true;
1412#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1413 /* Only remove the intercept if the nested-guest is also not intercepting it! */
1414 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1415 {
1416 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu, pCtx);
1417 fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1418 }
1419#else
1420 RT_NOREF2(pVCpu, pCtx);
1421#endif
1422 if (fRemove)
1423 {
1424 pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1425 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1426 }
1427 }
1428#else
1429 RT_NOREF3(pVCpu, pCtx, pVmcb);
1430#endif
1431}
1432
1433
1434/**
1435 * Sets a control intercept in the specified VMCB.
1436 *
1437 * @param pVmcb Pointer to the VM control block.
1438 * @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1439 */
1440DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1441{
1442 if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1443 {
1444 pVmcb->ctrl.u64InterceptCtrl |= fCtrlIntercept;
1445 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1446 }
1447}
1448
1449
1450/**
1451 * Clears a control intercept in the specified VMCB.
1452 *
1453 * @returns @c true if the intercept is still set, @c false otherwise.
1454 * @param pVCpu The cross context virtual CPU structure.
1455 * @param pCtx Pointer to the guest-CPU context.
1456 * @param pVmcb Pointer to the VM control block.
1457 * @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1458 *
1459 * @remarks This takes into account if we're executing a nested-guest and only
1460 * removes the control intercept if both the guest -and- nested-guest
1461 * are not intercepting it.
1462 */
1463DECLINLINE(bool) hmR0SvmClearCtrlIntercept(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1464{
1465 if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1466 {
1467 bool fRemove = true;
1468#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1469 /* Only remove the control intercept if the nested-guest is also not intercepting it! */
1470 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1471 {
1472 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu, pCtx);
1473 fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1474 }
1475#else
1476 RT_NOREF2(pVCpu, pCtx);
1477#endif
1478 if (fRemove)
1479 {
1480 pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1481 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1482 }
1483 }
1484
1485 return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1486}
1487
1488
1489/**
1490 * Loads the guest (or nested-guest) CR0 control register into the guest-state
1491 * area in the VMCB.
1492 *
1493 * Although the guest CR0 is a separate field in the VMCB we have to consider
1494 * the FPU state itself which is shared between the host and the guest.
1495 *
1496 * @returns VBox status code.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param pVmcb Pointer to the VM control block.
1499 * @param pCtx Pointer to the guest-CPU context.
1500 *
1501 * @remarks No-long-jump zone!!!
1502 */
1503static void hmR0SvmLoadSharedCR0(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1504{
1505 /* The guest FPU is now always pre-loaded before executing guest code, see @bugref{7243#c101}. */
1506 Assert(CPUMIsGuestFPUStateActive(pVCpu));
1507
1508 uint64_t const uGuestCr0 = pCtx->cr0;
1509 uint64_t uShadowCr0 = uGuestCr0;
1510
1511 /* Always enable caching. */
1512 uShadowCr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1513
1514 /* When Nested Paging is not available use shadow page tables and intercept #PFs (the latter done in SVMR0SetupVM()). */
1515 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1516 {
1517 uShadowCr0 |= X86_CR0_PG /* Use shadow page tables. */
1518 | X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1519 }
1520
1521 /*
1522 * Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that lets us
1523 * isolate the host from it, IEM/REM still needs work to emulate it properly. see @bugref{7243#c103}.
1524 */
1525 if (!(uGuestCr0 & X86_CR0_NE))
1526 {
1527 uShadowCr0 |= X86_CR0_NE;
1528 hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1529 }
1530 else
1531 hmR0SvmClearXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_MF);
1532
1533 /*
1534 * If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1535 *
1536 * CR0 writes still needs interception as PGM requires tracking paging mode changes, see @bugref{6944}.
1537 * We also don't ever want to honor weird things like cache disable from the guest. However, we can
1538 * avoid intercepting changes to the TS & MP bits by clearing the CR0 write intercept below and keeping
1539 * SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1540 */
1541 if (uShadowCr0 == uGuestCr0)
1542 {
1543 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1544 {
1545 pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1546 pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1547 Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1548 }
1549 else
1550 {
1551 /* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1552 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu, pCtx);
1553 pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1554 | (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1555 pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1556 | (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1557 }
1558 }
1559 else
1560 {
1561 pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(0);
1562 pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(0);
1563 }
1564 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1565
1566 Assert(RT_HI_U32(uShadowCr0) == 0);
1567 if (pVmcb->guest.u64CR0 != uShadowCr0)
1568 {
1569 pVmcb->guest.u64CR0 = uShadowCr0;
1570 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1571 }
1572}
1573
1574
1575/**
1576 * Loads the guest/nested-guest control registers (CR2, CR3, CR4) into the VMCB.
1577 *
1578 * @returns VBox status code.
1579 * @param pVCpu The cross context virtual CPU structure.
1580 * @param pVmcb Pointer to the VM control block.
1581 * @param pCtx Pointer to the guest-CPU context.
1582 *
1583 * @remarks No-long-jump zone!!!
1584 */
1585static int hmR0SvmLoadGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1586{
1587 PVM pVM = pVCpu->CTX_SUFF(pVM);
1588
1589 /*
1590 * Guest CR2.
1591 */
1592 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR2))
1593 {
1594 pVmcb->guest.u64CR2 = pCtx->cr2;
1595 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1596 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
1597 }
1598
1599 /*
1600 * Guest CR3.
1601 */
1602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
1603 {
1604 if (pVM->hm.s.fNestedPaging)
1605 {
1606 PGMMODE enmShwPagingMode;
1607#if HC_ARCH_BITS == 32
1608 if (CPUMIsGuestInLongModeEx(pCtx))
1609 enmShwPagingMode = PGMMODE_AMD64_NX;
1610 else
1611#endif
1612 enmShwPagingMode = PGMGetHostMode(pVM);
1613
1614 pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1615 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1616 Assert(pVmcb->ctrl.u64NestedPagingCR3);
1617 pVmcb->guest.u64CR3 = pCtx->cr3;
1618 }
1619 else
1620 {
1621 pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1622 Log4(("hmR0SvmLoadGuestControlRegs: CR3=%#RX64 (HyperCR3=%#RX64)\n", pCtx->cr3, pVmcb->guest.u64CR3));
1623 }
1624
1625 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1626 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
1627 }
1628
1629 /*
1630 * Guest CR4.
1631 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
1632 */
1633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
1634 {
1635 uint64_t uShadowCr4 = pCtx->cr4;
1636 if (!pVM->hm.s.fNestedPaging)
1637 {
1638 switch (pVCpu->hm.s.enmShadowMode)
1639 {
1640 case PGMMODE_REAL:
1641 case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1642 AssertFailed();
1643 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1644
1645 case PGMMODE_32_BIT: /* 32-bit paging. */
1646 uShadowCr4 &= ~X86_CR4_PAE;
1647 break;
1648
1649 case PGMMODE_PAE: /* PAE paging. */
1650 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1651 /** Must use PAE paging as we could use physical memory > 4 GB */
1652 uShadowCr4 |= X86_CR4_PAE;
1653 break;
1654
1655 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1656 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1657#ifdef VBOX_ENABLE_64_BITS_GUESTS
1658 break;
1659#else
1660 AssertFailed();
1661 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1662#endif
1663
1664 default: /* shut up gcc */
1665 AssertFailed();
1666 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1667 }
1668 }
1669
1670 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1671 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1672
1673 /* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1674 if (uShadowCr4 == pCtx->cr4)
1675 {
1676 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1677 pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1678 else
1679 {
1680 /* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1681 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu, pCtx);
1682 pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1683 | (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1684 }
1685 }
1686 else
1687 pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(4);
1688
1689 /* CR4 writes are always intercepted (both guest, nested-guest) from tracking PGM mode changes. */
1690 Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1691
1692 /* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1693 Assert(RT_HI_U32(uShadowCr4) == 0);
1694 pVmcb->guest.u64CR4 = uShadowCr4;
1695 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER | HMSVM_VMCB_CLEAN_INTERCEPTS);
1696
1697 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
1698 }
1699
1700 return VINF_SUCCESS;
1701}
1702
1703
1704/**
1705 * Loads the guest (or nested-guest) segment registers into the VMCB.
1706 *
1707 * @returns VBox status code.
1708 * @param pVCpu The cross context virtual CPU structure.
1709 * @param pVmcb Pointer to the VM control block.
1710 * @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
1711 *
1712 * @remarks No-long-jump zone!!!
1713 */
1714static void hmR0SvmLoadGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1715{
1716 /* Guest Segment registers: CS, SS, DS, ES, FS, GS. */
1717 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
1718 {
1719 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1720 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1721 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1722 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1723 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1724 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1725
1726 pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1727 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1728 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
1729 }
1730
1731 /* Guest TR. */
1732 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
1733 {
1734 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
1736 }
1737
1738 /* Guest LDTR. */
1739 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
1740 {
1741 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1742 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
1743 }
1744
1745 /* Guest GDTR. */
1746 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
1747 {
1748 pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1749 pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1750 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
1752 }
1753
1754 /* Guest IDTR. */
1755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
1756 {
1757 pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1758 pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1759 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
1761 }
1762}
1763
1764
1765/**
1766 * Loads the guest (or nested-guest) MSRs into the VMCB.
1767 *
1768 * @param pVCpu The cross context virtual CPU structure.
1769 * @param pVmcb Pointer to the VM control block.
1770 * @param pCtx Pointer to the guest-CPU context.
1771 *
1772 * @remarks No-long-jump zone!!!
1773 */
1774static void hmR0SvmLoadGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1775{
1776 /* Guest Sysenter MSRs. */
1777 pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1778 pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1779 pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1780
1781 /*
1782 * Guest EFER MSR.
1783 * AMD-V requires guest EFER.SVME to be set. Weird.
1784 * See AMD spec. 15.5.1 "Basic Operation" | "Canonicalization and Consistency Checks".
1785 */
1786 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
1787 {
1788 pVmcb->guest.u64EFER = pCtx->msrEFER | MSR_K6_EFER_SVME;
1789 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1790 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
1791 }
1792
1793 /* 64-bit MSRs. */
1794 if (CPUMIsGuestInLongModeEx(pCtx))
1795 {
1796 /* Load these always as the guest may modify FS/GS base using MSRs in 64-bit mode which we don't intercept. */
1797 pVmcb->guest.FS.u64Base = pCtx->fs.u64Base;
1798 pVmcb->guest.GS.u64Base = pCtx->gs.u64Base;
1799 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1800 }
1801 else
1802 {
1803 /* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit from guest EFER otherwise AMD-V expects amd64 shadow paging. */
1804 if (pCtx->msrEFER & MSR_K6_EFER_LME)
1805 {
1806 pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1807 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1808 }
1809 }
1810
1811 /** @todo The following are used in 64-bit only (SYSCALL/SYSRET) but they might
1812 * be writable in 32-bit mode. Clarify with AMD spec. */
1813 pVmcb->guest.u64STAR = pCtx->msrSTAR;
1814 pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1815 pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1816 pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1817 pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1818
1819 /*
1820 * Setup the PAT MSR (applicable for Nested Paging only).
1821 *
1822 * While guests can modify and see the modified values throug the shadow values,
1823 * we shall not honor any guest modifications of this MSR to ensure caching is always
1824 * enabled similar to how we always run with CR0.CD and NW bits cleared.
1825 *
1826 * For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1827 */
1828 pVmcb->guest.u64PAT = MSR_IA32_CR_PAT_INIT_VAL;
1829
1830 /* Enable the last branch record bit if LBR virtualization is enabled. */
1831 if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1832 pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1833}
1834
1835
1836/**
1837 * Loads the guest (or nested-guest) debug state into the VMCB and programs the
1838 * necessary intercepts accordingly.
1839 *
1840 * @param pVCpu The cross context virtual CPU structure.
1841 * @param pVmcb Pointer to the VM control block.
1842 * @param pCtx Pointer to the guest-CPU context.
1843 *
1844 * @remarks No-long-jump zone!!!
1845 * @remarks Requires EFLAGS to be up-to-date in the VMCB!
1846 */
1847static void hmR0SvmLoadSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1848{
1849 bool fInterceptMovDRx = false;
1850
1851 /*
1852 * Anyone single stepping on the host side? If so, we'll have to use the
1853 * trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1854 * the VMM level like the VT-x implementations does.
1855 */
1856 bool const fStepping = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
1857 if (fStepping)
1858 {
1859 pVCpu->hm.s.fClearTrapFlag = true;
1860 pVmcb->guest.u64RFlags |= X86_EFL_TF;
1861 fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1862 }
1863
1864 if ( fStepping
1865 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1866 {
1867 /*
1868 * Use the combined guest and host DRx values found in the hypervisor
1869 * register set because the debugger has breakpoints active or someone
1870 * is single stepping on the host side.
1871 *
1872 * Note! DBGF expects a clean DR6 state before executing guest code.
1873 */
1874#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1875 if ( CPUMIsGuestInLongModeEx(pCtx)
1876 && !CPUMIsHyperDebugStateActivePending(pVCpu))
1877 {
1878 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1879 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1880 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1881 }
1882 else
1883#endif
1884 if (!CPUMIsHyperDebugStateActive(pVCpu))
1885 {
1886 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1887 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1888 Assert(CPUMIsHyperDebugStateActive(pVCpu));
1889 }
1890
1891 /* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1892 if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1893 || pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1894 {
1895 pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1896 pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1897 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1898 pVCpu->hm.s.fUsingHyperDR7 = true;
1899 }
1900
1901 /** @todo If we cared, we could optimize to allow the guest to read registers
1902 * with the same values. */
1903 fInterceptMovDRx = true;
1904 Log5(("hmR0SvmLoadSharedDebugState: Loaded hyper DRx\n"));
1905 }
1906 else
1907 {
1908 /*
1909 * Update DR6, DR7 with the guest values if necessary.
1910 */
1911 if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1912 || pVmcb->guest.u64DR6 != pCtx->dr[6])
1913 {
1914 pVmcb->guest.u64DR7 = pCtx->dr[7];
1915 pVmcb->guest.u64DR6 = pCtx->dr[6];
1916 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1917 pVCpu->hm.s.fUsingHyperDR7 = false;
1918 }
1919
1920 /*
1921 * If the guest has enabled debug registers, we need to load them prior to
1922 * executing guest code so they'll trigger at the right time.
1923 */
1924 if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
1925 {
1926#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1927 if ( CPUMIsGuestInLongModeEx(pCtx)
1928 && !CPUMIsGuestDebugStateActivePending(pVCpu))
1929 {
1930 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1931 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1932 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
1933 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
1934 }
1935 else
1936#endif
1937 if (!CPUMIsGuestDebugStateActive(pVCpu))
1938 {
1939 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1940 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1941 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1942 Assert(CPUMIsGuestDebugStateActive(pVCpu));
1943 }
1944 Log5(("hmR0SvmLoadSharedDebugState: Loaded guest DRx\n"));
1945 }
1946 /*
1947 * If no debugging enabled, we'll lazy load DR0-3. We don't need to
1948 * intercept #DB as DR6 is updated in the VMCB.
1949 *
1950 * Note! If we cared and dared, we could skip intercepting \#DB here.
1951 * However, \#DB shouldn't be performance critical, so we'll play safe
1952 * and keep the code similar to the VT-x code and always intercept it.
1953 */
1954#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1955 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
1956 && !CPUMIsGuestDebugStateActive(pVCpu))
1957#else
1958 else if (!CPUMIsGuestDebugStateActive(pVCpu))
1959#endif
1960 {
1961 fInterceptMovDRx = true;
1962 }
1963 }
1964
1965 Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
1966 if (fInterceptMovDRx)
1967 {
1968 if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1969 || pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1970 {
1971 pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
1972 pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
1973 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1974 }
1975 }
1976 else
1977 {
1978 if ( pVmcb->ctrl.u16InterceptRdDRx
1979 || pVmcb->ctrl.u16InterceptWrDRx)
1980 {
1981 pVmcb->ctrl.u16InterceptRdDRx = 0;
1982 pVmcb->ctrl.u16InterceptWrDRx = 0;
1983 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1984 }
1985 }
1986 Log4(("hmR0SvmLoadSharedDebugState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
1987}
1988
1989
1990#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1991/**
1992 * Loads the nested-guest APIC state (currently just the TPR).
1993 *
1994 * @param pVCpu The cross context virtual CPU structure.
1995 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
1996 */
1997static void hmR0SvmLoadGuestApicStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst)
1998{
1999 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
2000 {
2001 Assert(pVmcbNstGst->ctrl.IntCtrl.n.u1VIntrMasking == 1); RT_NOREF(pVmcbNstGst);
2002 pVCpu->hm.s.svm.fSyncVTpr = false;
2003 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
2004 }
2005}
2006
2007
2008/**
2009 * Loads the nested-guest hardware virtualization state.
2010 *
2011 * @param pVCpu The cross context virtual CPU structure.
2012 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
2013 * @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
2014 */
2015static void hmR0SvmLoadGuestHwvirtStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst, PCPUMCTX pCtx)
2016{
2017 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_HWVIRT))
2018 {
2019 /*
2020 * Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2021 * since SVM doesn't have a preemption timer.
2022 *
2023 * We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2024 * nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2025 * and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2026 */
2027 PVM pVM = pVCpu->CTX_SUFF(pVM);
2028 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2029 uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2030 uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2031 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
2032 {
2033 pVmcbNstGstCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2034 pVmcbNstGstCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2035 pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2036 }
2037 else
2038 {
2039 pVmcbNstGstCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2040 pVmcbNstGstCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2041 }
2042
2043 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_HWVIRT);
2044 }
2045}
2046#endif
2047
2048/**
2049 * Loads the guest APIC state (currently just the TPR).
2050 *
2051 * @returns VBox status code.
2052 * @param pVCpu The cross context virtual CPU structure.
2053 * @param pVmcb Pointer to the VM control block.
2054 * @param pCtx Pointer to the guest-CPU context.
2055 */
2056static int hmR0SvmLoadGuestApicState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
2057{
2058 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
2059 return VINF_SUCCESS;
2060
2061 int rc = VINF_SUCCESS;
2062 PVM pVM = pVCpu->CTX_SUFF(pVM);
2063 if ( PDMHasApic(pVM)
2064 && APICIsEnabled(pVCpu))
2065 {
2066 bool fPendingIntr;
2067 uint8_t u8Tpr;
2068 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2069 AssertRCReturn(rc, rc);
2070
2071 /* Assume that we need to trap all TPR accesses and thus need not check on
2072 every #VMEXIT if we should update the TPR. */
2073 Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2074 pVCpu->hm.s.svm.fSyncVTpr = false;
2075
2076 /* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2077 if (pVM->hm.s.fTPRPatchingActive)
2078 {
2079 pCtx->msrLSTAR = u8Tpr;
2080 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
2081
2082 /* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2083 if (fPendingIntr)
2084 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2085 else
2086 {
2087 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2088 pVCpu->hm.s.svm.fSyncVTpr = true;
2089 }
2090 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2091 }
2092 else
2093 {
2094 /* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2095 pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2096
2097 /* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we can deliver the interrupt to the guest. */
2098 if (fPendingIntr)
2099 pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(8);
2100 else
2101 {
2102 pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2103 pVCpu->hm.s.svm.fSyncVTpr = true;
2104 }
2105
2106 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS | HMSVM_VMCB_CLEAN_INT_CTRL);
2107 }
2108 }
2109
2110 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
2111 return rc;
2112}
2113
2114
2115/**
2116 * Loads the exception interrupts required for guest (or nested-guest) execution in
2117 * the VMCB.
2118 *
2119 * @param pVCpu The cross context virtual CPU structure.
2120 * @param pVmcb Pointer to the VM control block.
2121 * @param pCtx Pointer to the guest-CPU context.
2122 */
2123static void hmR0SvmLoadGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
2124{
2125 /* If we modify intercepts from here, please check & adjust hmR0SvmLoadGuestXcptInterceptsNested()
2126 if required. */
2127 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
2128 {
2129 /* Trap #UD for GIM provider (e.g. for hypercalls). */
2130 if (pVCpu->hm.s.fGIMTrapXcptUD)
2131 hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2132 else
2133 hmR0SvmClearXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_UD);
2134
2135 /* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2136 if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2137 hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2138 else
2139 hmR0SvmClearXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_BP);
2140
2141 /* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmLoadSharedCR0(). */
2142 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
2143 }
2144}
2145
2146
2147#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2148/**
2149 * Merges guest and nested-guest intercepts for executing the nested-guest using
2150 * hardware-assisted SVM.
2151 *
2152 * This merges the guest and nested-guest intercepts in a way that if the outer
2153 * guest intercept is set we need to intercept it in the nested-guest as
2154 * well.
2155 *
2156 * @param pVCpu The cross context virtual CPU structure.
2157 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
2158 * @param pCtx Pointer to the nested-guest-CPU context.
2159 */
2160static void hmR0SvmMergeVmcbCtrlsNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2161{
2162 PVM pVM = pVCpu->CTX_SUFF(pVM);
2163 PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2164 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2165 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2166
2167 /* Merge the guest's CR intercepts into the nested-guest VMCB. */
2168 pVmcbNstGstCtrl->u16InterceptRdCRx |= pVmcb->ctrl.u16InterceptRdCRx;
2169 pVmcbNstGstCtrl->u16InterceptWrCRx |= pVmcb->ctrl.u16InterceptWrCRx;
2170
2171 /* Always intercept CR4 writes for tracking PGM mode changes. */
2172 pVmcbNstGstCtrl->u16InterceptWrCRx |= RT_BIT(4);
2173
2174 /* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2175 if (!pVM->hm.s.fNestedPaging)
2176 {
2177 pVmcbNstGstCtrl->u16InterceptRdCRx |= RT_BIT(3);
2178 pVmcbNstGstCtrl->u16InterceptWrCRx |= RT_BIT(3);
2179 }
2180
2181 /** @todo Figure out debugging with nested-guests, till then just intercept
2182 * all DR[0-15] accesses. */
2183 pVmcbNstGstCtrl->u16InterceptRdDRx |= 0xffff;
2184 pVmcbNstGstCtrl->u16InterceptWrDRx |= 0xffff;
2185
2186 /*
2187 * Merge the guest's exception intercepts into the nested-guest VMCB.
2188 *
2189 * - \#UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2190 * while executing the nested-guest.
2191 *
2192 * - \#BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2193 * be tweaked later depending on how we wish to implement breakpoints.
2194 *
2195 * Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2196 * for VM debugger breakpoints, see hmR0SvmLoadGuestXcptIntercepts.
2197 */
2198#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2199 pVmcbNstGstCtrl->u32InterceptXcpt |= (pVmcb->ctrl.u32InterceptXcpt & ~( RT_BIT(X86_XCPT_UD)
2200 | RT_BIT(X86_XCPT_BP)));
2201#else
2202 pVmcbNstGstCtrl->u32InterceptXcpt |= pVmcb->ctrl.u32InterceptXcpt;
2203#endif
2204
2205 /*
2206 * Adjust intercepts while executing the nested-guest that differ from the
2207 * outer guest intercepts.
2208 *
2209 * - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2210 * that belong to the nested-guest to the outer guest.
2211 *
2212 * - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2213 * the nested-guest, the physical CPU raises a \#UD exception as expected.
2214 */
2215 pVmcbNstGstCtrl->u64InterceptCtrl |= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2216 | SVM_CTRL_INTERCEPT_VMMCALL))
2217 | HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2218
2219 Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2220 == HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2221
2222 /* Finally, update the VMCB clean bits. */
2223 pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2224}
2225#endif
2226
2227
2228/**
2229 * Sets up the appropriate function to run guest code.
2230 *
2231 * @returns VBox status code.
2232 * @param pVCpu The cross context virtual CPU structure.
2233 *
2234 * @remarks No-long-jump zone!!!
2235 */
2236static int hmR0SvmSetupVMRunHandler(PVMCPU pVCpu)
2237{
2238 if (CPUMIsGuestInLongMode(pVCpu))
2239 {
2240#ifndef VBOX_ENABLE_64_BITS_GUESTS
2241 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2242#endif
2243 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
2244#if HC_ARCH_BITS == 32
2245 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
2246 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
2247#else
2248 /* 64-bit host or hybrid host. */
2249 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
2250#endif
2251 }
2252 else
2253 {
2254 /* Guest is not in long mode, use the 32-bit handler. */
2255 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
2256 }
2257 return VINF_SUCCESS;
2258}
2259
2260
2261/**
2262 * Enters the AMD-V session.
2263 *
2264 * @returns VBox status code.
2265 * @param pVM The cross context VM structure.
2266 * @param pVCpu The cross context virtual CPU structure.
2267 * @param pCpu Pointer to the CPU info struct.
2268 */
2269VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2270{
2271 AssertPtr(pVM);
2272 AssertPtr(pVCpu);
2273 Assert(pVM->hm.s.svm.fSupported);
2274 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2275 NOREF(pVM); NOREF(pCpu);
2276
2277 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2278 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
2279
2280 pVCpu->hm.s.fLeaveDone = false;
2281 return VINF_SUCCESS;
2282}
2283
2284
2285/**
2286 * Thread-context callback for AMD-V.
2287 *
2288 * @param enmEvent The thread-context event.
2289 * @param pVCpu The cross context virtual CPU structure.
2290 * @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2291 * @thread EMT(pVCpu)
2292 */
2293VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
2294{
2295 NOREF(fGlobalInit);
2296
2297 switch (enmEvent)
2298 {
2299 case RTTHREADCTXEVENT_OUT:
2300 {
2301 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2302 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2303 VMCPU_ASSERT_EMT(pVCpu);
2304
2305 /* No longjmps (log-flush, locks) in this fragile context. */
2306 VMMRZCallRing3Disable(pVCpu);
2307
2308 if (!pVCpu->hm.s.fLeaveDone)
2309 {
2310 hmR0SvmLeave(pVCpu);
2311 pVCpu->hm.s.fLeaveDone = true;
2312 }
2313
2314 /* Leave HM context, takes care of local init (term). */
2315 int rc = HMR0LeaveCpu(pVCpu);
2316 AssertRC(rc); NOREF(rc);
2317
2318 /* Restore longjmp state. */
2319 VMMRZCallRing3Enable(pVCpu);
2320 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2321 break;
2322 }
2323
2324 case RTTHREADCTXEVENT_IN:
2325 {
2326 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2327 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2328 VMCPU_ASSERT_EMT(pVCpu);
2329
2330 /* No longjmps (log-flush, locks) in this fragile context. */
2331 VMMRZCallRing3Disable(pVCpu);
2332
2333 /*
2334 * Initialize the bare minimum state required for HM. This takes care of
2335 * initializing AMD-V if necessary (onlined CPUs, local init etc.)
2336 */
2337 int rc = HMR0EnterCpu(pVCpu);
2338 AssertRC(rc); NOREF(rc);
2339 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
2340
2341 pVCpu->hm.s.fLeaveDone = false;
2342
2343 /* Restore longjmp state. */
2344 VMMRZCallRing3Enable(pVCpu);
2345 break;
2346 }
2347
2348 default:
2349 break;
2350 }
2351}
2352
2353
2354/**
2355 * Saves the host state.
2356 *
2357 * @returns VBox status code.
2358 * @param pVM The cross context VM structure.
2359 * @param pVCpu The cross context virtual CPU structure.
2360 *
2361 * @remarks No-long-jump zone!!!
2362 */
2363VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
2364{
2365 NOREF(pVM);
2366 NOREF(pVCpu);
2367 /* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
2369 return VINF_SUCCESS;
2370}
2371
2372
2373/**
2374 * Loads the guest state into the VMCB.
2375 *
2376 * The CPU state will be loaded from these fields on every successful VM-entry.
2377 * Also sets up the appropriate VMRUN function to execute guest code based on
2378 * the guest CPU mode.
2379 *
2380 * @returns VBox status code.
2381 * @param pVM The cross context VM structure.
2382 * @param pVCpu The cross context virtual CPU structure.
2383 * @param pCtx Pointer to the guest-CPU context.
2384 *
2385 * @remarks No-long-jump zone!!!
2386 */
2387static int hmR0SvmLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2388{
2389 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
2390
2391 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2392 AssertMsgReturn(pVmcb, ("Invalid pVmcb\n"), VERR_SVM_INVALID_PVMCB);
2393
2394 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2395
2396 int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcb, pCtx);
2397 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestControlRegs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2398
2399 hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcb, pCtx);
2400 hmR0SvmLoadGuestMsrs(pVCpu, pVmcb, pCtx);
2401
2402 pVmcb->guest.u64RIP = pCtx->rip;
2403 pVmcb->guest.u64RSP = pCtx->rsp;
2404 pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2405 pVmcb->guest.u64RAX = pCtx->rax;
2406
2407#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2408 if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2409 {
2410 Assert(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VGIF);
2411 pVmcb->ctrl.IntCtrl.n.u1VGif = pCtx->hwvirt.fGif;
2412 }
2413#endif
2414
2415 rc = hmR0SvmLoadGuestApicState(pVCpu, pVmcb, pCtx);
2416 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2417
2418 hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
2419
2420 rc = hmR0SvmSetupVMRunHandler(pVCpu);
2421 AssertLogRelMsgRCReturn(rc, ("hmR0SvmSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2422
2423 /* Clear any unused and reserved bits. */
2424 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2425 | HM_CHANGED_GUEST_RSP
2426 | HM_CHANGED_GUEST_RFLAGS
2427 | HM_CHANGED_GUEST_SYSENTER_CS_MSR
2428 | HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2429 | HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2430 | HM_CHANGED_GUEST_HWVIRT /* Unused. */
2431 | HM_CHANGED_VMM_GUEST_LAZY_MSRS
2432 | HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2433 | HM_CHANGED_SVM_RESERVED2
2434 | HM_CHANGED_SVM_RESERVED3
2435 | HM_CHANGED_SVM_RESERVED4);
2436
2437 /* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2438 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2439 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
2440 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2441
2442#ifdef VBOX_STRICT
2443 hmR0SvmLogState(pVCpu, pVmcb, pCtx, "hmR0SvmLoadGuestState", 0 /* fFlags */, 0 /* uVerbose */);
2444#endif
2445 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2446 return rc;
2447}
2448
2449
2450#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2451/**
2452 * Merges the guest and nested-guest MSR permission bitmap.
2453 *
2454 * If the guest is intercepting an MSR we need to intercept it regardless of
2455 * whether the nested-guest is intercepting it or not.
2456 *
2457 * @param pHostCpu Pointer to the physical CPU HM info. struct.
2458 * @param pVCpu The cross context virtual CPU structure.
2459 * @param pCtx Pointer to the nested-guest-CPU context.
2460 */
2461static void hmR0SvmMergeMsrpmNested(PHMGLOBALCPUINFO pHostCpu, PVMCPU pVCpu, PCPUMCTX pCtx)
2462{
2463 uint64_t const *pu64GstMsrpm = (uint64_t const *)pVCpu->hm.s.svm.pvMsrBitmap;
2464 uint64_t const *pu64NstGstMsrpm = (uint64_t const *)pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
2465 uint64_t *pu64DstMsrpm = (uint64_t *)pHostCpu->n.svm.pvNstGstMsrpm;
2466
2467 /* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2468 uint32_t const offRsvdQwords = 0x1800 >> 3;
2469 for (uint32_t i = 0; i < offRsvdQwords; i++)
2470 pu64DstMsrpm[i] = pu64NstGstMsrpm[i] | pu64GstMsrpm[i];
2471}
2472
2473
2474/**
2475 * Caches the nested-guest VMCB fields before we modify them for execution using
2476 * hardware-assisted SVM.
2477 *
2478 * @returns true if the VMCB was previously already cached, false otherwise.
2479 * @param pCtx Pointer to the guest-CPU context.
2480 *
2481 * @sa HMSvmNstGstVmExitNotify.
2482 */
2483static bool hmR0SvmCacheVmcbNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2484{
2485 /*
2486 * Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2487 * Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2488 *
2489 * Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2490 * cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2491 */
2492 bool const fWasCached = pCtx->hwvirt.svm.fHMCachedVmcb;
2493 if (!fWasCached)
2494 {
2495 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2496 PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2497 PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2498
2499 pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2500 pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2501 pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2502 pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2503 pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2504 pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2505 pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2506 pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2507 pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2508 pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2509 pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2510 pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2511 pCtx->hwvirt.svm.fHMCachedVmcb = true;
2512 Log4(("hmR0SvmCacheVmcbNested: Cached VMCB fields\n"));
2513 }
2514
2515 return fWasCached;
2516}
2517
2518
2519/**
2520 * Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2521 *
2522 * This is done the first time we enter nested-guest execution using SVM R0
2523 * until the nested-guest \#VMEXIT (not to be confused with physical CPU
2524 * \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2525 *
2526 * @param pVCpu The cross context virtual CPU structure.
2527 * @param pCtx Pointer to the nested-guest-CPU context.
2528 */
2529static void hmR0SvmSetupVmcbNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2530{
2531 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2532 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2533
2534 /*
2535 * First cache the nested-guest VMCB fields we may potentially modify.
2536 */
2537 bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu, pCtx);
2538 if (!fVmcbCached)
2539 {
2540 /*
2541 * The IOPM of the nested-guest can be ignored because the the guest always
2542 * intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2543 * than the nested-guest IOPM and swap the field back on the #VMEXIT.
2544 */
2545 pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2546
2547 /*
2548 * Use the same nested-paging as the outer guest. We can't dynamically switch off
2549 * nested-paging suddenly while executing a VM (see assertion at the end of
2550 * Trap0eHandler() in PGMAllBth.h).
2551 */
2552 pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
2553
2554 /* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2555 pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2556
2557#ifdef DEBUG_ramshankar
2558 /* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2559 pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2560#endif
2561
2562 /*
2563 * If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2564 * need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2565 */
2566 if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2567 pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
2568 | SVM_CTRL_INTERCEPT_VMLOAD;
2569
2570 /*
2571 * If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2572 * CLGI/STGI instructions executed by the nested-guest.
2573 */
2574 if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2575 pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_CLGI
2576 | SVM_CTRL_INTERCEPT_STGI;
2577
2578 /* Merge the guest and nested-guest intercepts. */
2579 hmR0SvmMergeVmcbCtrlsNested(pVCpu, pCtx);
2580
2581 /* Update the VMCB clean bits. */
2582 pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2583 }
2584 else
2585 {
2586 Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2587 Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
2588 }
2589}
2590
2591
2592/**
2593 * Loads the nested-guest state into the VMCB.
2594 *
2595 * @returns VBox status code.
2596 * @param pVCpu The cross context virtual CPU structure.
2597 * @param pCtx Pointer to the guest-CPU context.
2598 *
2599 * @remarks No-long-jump zone!!!
2600 */
2601static int hmR0SvmLoadGuestStateNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2602{
2603 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2604
2605 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2606 Assert(pVmcbNstGst);
2607
2608 hmR0SvmSetupVmcbNested(pVCpu, pCtx);
2609
2610 int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcbNstGst, pCtx);
2611 AssertRCReturn(rc, rc);
2612
2613 /*
2614 * We need to load the entire state (including FS, GS etc.) as we could be continuing
2615 * to execute the nested-guest at any point (not just immediately after VMRUN) and thus
2616 * the VMCB can possibly be out-of-sync with the actual nested-guest state if it was
2617 * executed in IEM.
2618 */
2619 hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcbNstGst, pCtx);
2620 hmR0SvmLoadGuestMsrs(pVCpu, pVmcbNstGst, pCtx);
2621 hmR0SvmLoadGuestApicStateNested(pVCpu, pVmcbNstGst);
2622 hmR0SvmLoadGuestHwvirtStateNested(pVCpu, pVmcbNstGst, pCtx);
2623
2624 pVmcbNstGst->guest.u64RIP = pCtx->rip;
2625 pVmcbNstGst->guest.u64RSP = pCtx->rsp;
2626 pVmcbNstGst->guest.u64RFlags = pCtx->eflags.u32;
2627 pVmcbNstGst->guest.u64RAX = pCtx->rax;
2628
2629#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2630 Assert(!pVmcbNstGst->ctrl.IntCtrl.n.u1VGifEnable); /* Nested VGIF not supported yet. */
2631#endif
2632
2633 rc = hmR0SvmSetupVMRunHandler(pVCpu);
2634 AssertRCReturn(rc, rc);
2635
2636 /* Clear any unused and reserved bits. */
2637 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2638 | HM_CHANGED_GUEST_RSP
2639 | HM_CHANGED_GUEST_RFLAGS
2640 | HM_CHANGED_GUEST_SYSENTER_CS_MSR
2641 | HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2642 | HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2643 | HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS /* Unused. */
2644 | HM_CHANGED_VMM_GUEST_LAZY_MSRS
2645 | HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2646 | HM_CHANGED_SVM_RESERVED2
2647 | HM_CHANGED_SVM_RESERVED3
2648 | HM_CHANGED_SVM_RESERVED4);
2649
2650 /* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2651 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2652 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
2653 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2654
2655#ifdef VBOX_STRICT
2656 hmR0SvmLogState(pVCpu, pVmcbNstGst, pCtx, "hmR0SvmLoadGuestStateNested", HMSVM_LOG_ALL, 0 /* uVerbose */);
2657#endif
2658 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2659 return rc;
2660}
2661#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2662
2663
2664/**
2665 * Loads the state shared between the host and guest (or nested-guest) into the
2666 * VMCB.
2667 *
2668 * @param pVCpu The cross context virtual CPU structure.
2669 * @param pVmcb Pointer to the VM control block.
2670 * @param pCtx Pointer to the guest-CPU context.
2671 *
2672 * @remarks No-long-jump zone!!!
2673 */
2674static void hmR0SvmLoadSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
2675{
2676 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2677 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2678
2679 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
2680 {
2681 hmR0SvmLoadSharedCR0(pVCpu, pVmcb, pCtx);
2682 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
2683 }
2684
2685 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
2686 {
2687 /** @todo Figure out stepping with nested-guest. */
2688 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2689 hmR0SvmLoadSharedDebugState(pVCpu, pVmcb, pCtx);
2690 else
2691 {
2692 pVmcb->guest.u64DR6 = pCtx->dr[6];
2693 pVmcb->guest.u64DR7 = pCtx->dr[7];
2694 Log4(("hmR0SvmLoadSharedState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2695 }
2696
2697 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
2698 }
2699
2700 /* Unused on AMD-V (no lazy MSRs). */
2701 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
2702
2703 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
2704 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2705}
2706
2707
2708/**
2709 * Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2710 * context.
2711 *
2712 * Currently there is no residual state left in the CPU that is not updated in the
2713 * VMCB.
2714 *
2715 * @returns VBox status code.
2716 * @param pVCpu The cross context virtual CPU structure.
2717 * @param pMixedCtx Pointer to the guest-CPU or nested-guest-CPU
2718 * context. The data may be out-of-sync. Make sure to
2719 * update the required fields before using them.
2720 * @param pVmcb Pointer to the VM control block.
2721 */
2722static void hmR0SvmSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PCSVMVMCB pVmcb)
2723{
2724 Assert(VMMRZCallRing3IsEnabled(pVCpu));
2725
2726 pMixedCtx->rip = pVmcb->guest.u64RIP;
2727 pMixedCtx->rsp = pVmcb->guest.u64RSP;
2728 pMixedCtx->eflags.u32 = pVmcb->guest.u64RFlags;
2729 pMixedCtx->rax = pVmcb->guest.u64RAX;
2730
2731 PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2732#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2733 if (!CPUMIsGuestInSvmNestedHwVirtMode(pMixedCtx))
2734 {
2735 if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2736 {
2737 /*
2738 * Guest Virtual GIF (Global Interrupt Flag).
2739 * We don't yet support passing VGIF feature to the guest.
2740 */
2741 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fVGif);
2742 pMixedCtx->hwvirt.fGif = pVmcbCtrl->IntCtrl.n.u1VGif;
2743 }
2744 }
2745 else
2746 {
2747 /*
2748 * Nested-guest interrupt pending.
2749 * Sync nested-guest's V_IRQ and its force-flag.
2750 */
2751 if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2752 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2753 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2754 }
2755#endif
2756
2757 /*
2758 * Guest interrupt shadow.
2759 */
2760 if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2761 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
2762 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2763 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2764
2765 /*
2766 * Guest control registers: CR0, CR2, CR3 (handled at the end).
2767 * Accesses to other control registers are always intercepted.
2768 */
2769 pMixedCtx->cr2 = pVmcb->guest.u64CR2;
2770
2771 /* If we're not intercepting changes to CR0 TS & MP bits, sync those bits here. */
2772 if (!(pVmcbCtrl->u16InterceptWrCRx & RT_BIT(0)))
2773 {
2774 pMixedCtx->cr0 = (pMixedCtx->cr0 & ~(X86_CR0_TS | X86_CR0_MP))
2775 | (pVmcb->guest.u64CR0 & (X86_CR0_TS | X86_CR0_MP));
2776 }
2777
2778 /*
2779 * Guest MSRs.
2780 */
2781 pMixedCtx->msrSTAR = pVmcb->guest.u64STAR; /* legacy syscall eip, cs & ss */
2782 pMixedCtx->msrLSTAR = pVmcb->guest.u64LSTAR; /* 64-bit mode syscall rip */
2783 pMixedCtx->msrCSTAR = pVmcb->guest.u64CSTAR; /* compatibility mode syscall rip */
2784 pMixedCtx->msrSFMASK = pVmcb->guest.u64SFMASK; /* syscall flag mask */
2785 pMixedCtx->msrKERNELGSBASE = pVmcb->guest.u64KernelGSBase; /* swapgs exchange value */
2786 pMixedCtx->SysEnter.cs = pVmcb->guest.u64SysEnterCS;
2787 pMixedCtx->SysEnter.eip = pVmcb->guest.u64SysEnterEIP;
2788 pMixedCtx->SysEnter.esp = pVmcb->guest.u64SysEnterESP;
2789
2790 /*
2791 * Guest segment registers (includes FS, GS base MSRs for 64-bit guests).
2792 */
2793 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, CS, cs);
2794 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, SS, ss);
2795 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, DS, ds);
2796 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, ES, es);
2797 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, FS, fs);
2798 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, GS, gs);
2799
2800 /*
2801 * Correct the hidden CS granularity bit. Haven't seen it being wrong in any other
2802 * register (yet).
2803 */
2804 /** @todo SELM might need to be fixed as it too should not care about the
2805 * granularity bit. See @bugref{6785}. */
2806 if ( !pMixedCtx->cs.Attr.n.u1Granularity
2807 && pMixedCtx->cs.Attr.n.u1Present
2808 && pMixedCtx->cs.u32Limit > UINT32_C(0xfffff))
2809 {
2810 Assert((pMixedCtx->cs.u32Limit & 0xfff) == 0xfff);
2811 pMixedCtx->cs.Attr.n.u1Granularity = 1;
2812 }
2813
2814 HMSVM_ASSERT_SEG_GRANULARITY(cs);
2815 HMSVM_ASSERT_SEG_GRANULARITY(ss);
2816 HMSVM_ASSERT_SEG_GRANULARITY(ds);
2817 HMSVM_ASSERT_SEG_GRANULARITY(es);
2818 HMSVM_ASSERT_SEG_GRANULARITY(fs);
2819 HMSVM_ASSERT_SEG_GRANULARITY(gs);
2820
2821 /*
2822 * Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the VMCB and uses that
2823 * and thus it's possible that when the CPL changes during guest execution that the SS DPL
2824 * isn't updated by AMD-V. Observed on some AMD Fusion CPUs with 64-bit guests.
2825 * See AMD spec. 15.5.1 "Basic operation".
2826 */
2827 Assert(!(pVmcb->guest.u8CPL & ~0x3));
2828 uint8_t const uCpl = pVmcb->guest.u8CPL;
2829 if (pMixedCtx->ss.Attr.n.u2Dpl != uCpl)
2830 {
2831 Log4(("hmR0SvmSaveGuestState: CPL differs. SS.DPL=%u, CPL=%u, overwriting SS.DPL!\n", pMixedCtx->ss.Attr.n.u2Dpl, uCpl));
2832 pMixedCtx->ss.Attr.n.u2Dpl = pVmcb->guest.u8CPL & 0x3;
2833 }
2834
2835 /*
2836 * Guest TR.
2837 * Fixup TR attributes so it's compatible with Intel. Important when saved-states are used
2838 * between Intel and AMD. See @bugref{6208#c39}.
2839 * ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2840 */
2841 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, TR, tr);
2842 if (pMixedCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2843 {
2844 if ( pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2845 || CPUMIsGuestInLongModeEx(pMixedCtx))
2846 pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2847 else if (pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2848 pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2849 }
2850
2851 /*
2852 * Guest Descriptor-Table registers (GDTR, IDTR, LDTR).
2853 */
2854 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, LDTR, ldtr);
2855 pMixedCtx->gdtr.cbGdt = pVmcb->guest.GDTR.u32Limit;
2856 pMixedCtx->gdtr.pGdt = pVmcb->guest.GDTR.u64Base;
2857
2858 pMixedCtx->idtr.cbIdt = pVmcb->guest.IDTR.u32Limit;
2859 pMixedCtx->idtr.pIdt = pVmcb->guest.IDTR.u64Base;
2860
2861 /*
2862 * Guest Debug registers.
2863 */
2864 if (!pVCpu->hm.s.fUsingHyperDR7)
2865 {
2866 pMixedCtx->dr[6] = pVmcb->guest.u64DR6;
2867 pMixedCtx->dr[7] = pVmcb->guest.u64DR7;
2868 }
2869 else
2870 {
2871 Assert(pVmcb->guest.u64DR7 == CPUMGetHyperDR7(pVCpu));
2872 CPUMSetHyperDR6(pVCpu, pVmcb->guest.u64DR6);
2873 }
2874
2875 /*
2876 * With Nested Paging, CR3 changes are not intercepted. Therefore, sync. it now.
2877 * This is done as the very last step of syncing the guest state, as PGMUpdateCR3() may cause longjmp's to ring-3.
2878 */
2879 if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2880 && pMixedCtx->cr3 != pVmcb->guest.u64CR3)
2881 {
2882 CPUMSetGuestCR3(pVCpu, pVmcb->guest.u64CR3);
2883 PGMUpdateCR3(pVCpu, pVmcb->guest.u64CR3);
2884 }
2885
2886#ifdef VBOX_STRICT
2887 if (CPUMIsGuestInSvmNestedHwVirtMode(pMixedCtx))
2888 hmR0SvmLogState(pVCpu, pVmcb, pMixedCtx, "hmR0SvmSaveGuestStateNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR, 0 /* uVerbose */);
2889#endif
2890}
2891
2892
2893/**
2894 * Does the necessary state syncing before returning to ring-3 for any reason
2895 * (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2896 *
2897 * @param pVCpu The cross context virtual CPU structure.
2898 *
2899 * @remarks No-long-jmp zone!!!
2900 */
2901static void hmR0SvmLeave(PVMCPU pVCpu)
2902{
2903 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2904 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2905 Assert(VMMR0IsLogFlushDisabled(pVCpu));
2906
2907 /*
2908 * !!! IMPORTANT !!!
2909 * If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2910 */
2911
2912 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
2913 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
2914 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0); /** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2915
2916 /*
2917 * Restore host debug registers if necessary and resync on next R0 reentry.
2918 */
2919#ifdef VBOX_STRICT
2920 if (CPUMIsHyperDebugStateActive(pVCpu))
2921 {
2922 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
2923 Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2924 Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2925 }
2926#endif
2927 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */))
2928 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);/** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2929
2930 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2931 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2932
2933 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2934 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
2935 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2936 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2937 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2938
2939 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
2940}
2941
2942
2943/**
2944 * Leaves the AMD-V session.
2945 *
2946 * @returns VBox status code.
2947 * @param pVCpu The cross context virtual CPU structure.
2948 */
2949static int hmR0SvmLeaveSession(PVMCPU pVCpu)
2950{
2951 HM_DISABLE_PREEMPT();
2952 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2953 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2954
2955 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
2956 and done this from the SVMR0ThreadCtxCallback(). */
2957 if (!pVCpu->hm.s.fLeaveDone)
2958 {
2959 hmR0SvmLeave(pVCpu);
2960 pVCpu->hm.s.fLeaveDone = true;
2961 }
2962
2963 /*
2964 * !!! IMPORTANT !!!
2965 * If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2966 */
2967
2968 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2969 /* Deregister hook now that we've left HM context before re-enabling preemption. */
2970 VMMR0ThreadCtxHookDisable(pVCpu);
2971
2972 /* Leave HM context. This takes care of local init (term). */
2973 int rc = HMR0LeaveCpu(pVCpu);
2974
2975 HM_RESTORE_PREEMPT();
2976 return rc;
2977}
2978
2979
2980/**
2981 * Does the necessary state syncing before doing a longjmp to ring-3.
2982 *
2983 * @returns VBox status code.
2984 * @param pVCpu The cross context virtual CPU structure.
2985 *
2986 * @remarks No-long-jmp zone!!!
2987 */
2988static int hmR0SvmLongJmpToRing3(PVMCPU pVCpu)
2989{
2990 return hmR0SvmLeaveSession(pVCpu);
2991}
2992
2993
2994/**
2995 * VMMRZCallRing3() callback wrapper which saves the guest state (or restores
2996 * any remaining host state) before we longjump to ring-3 and possibly get
2997 * preempted.
2998 *
2999 * @param pVCpu The cross context virtual CPU structure.
3000 * @param enmOperation The operation causing the ring-3 longjump.
3001 * @param pvUser The user argument (pointer to the possibly
3002 * out-of-date guest-CPU context).
3003 */
3004static DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
3005{
3006 RT_NOREF_PV(pvUser);
3007
3008 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
3009 {
3010 /*
3011 * !!! IMPORTANT !!!
3012 * If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3013 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3014 */
3015 VMMRZCallRing3RemoveNotification(pVCpu);
3016 VMMRZCallRing3Disable(pVCpu);
3017 HM_DISABLE_PREEMPT();
3018
3019 /* Restore host FPU state if necessary and resync on next R0 reentry. */
3020 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3021
3022 /* Restore host debug registers if necessary and resync on next R0 reentry. */
3023 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3024
3025 /* Deregister the hook now that we've left HM context before re-enabling preemption. */
3026 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3027 VMMR0ThreadCtxHookDisable(pVCpu);
3028
3029 /* Leave HM context. This takes care of local init (term). */
3030 HMR0LeaveCpu(pVCpu);
3031
3032 HM_RESTORE_PREEMPT();
3033 return VINF_SUCCESS;
3034 }
3035
3036 Assert(pVCpu);
3037 Assert(pvUser);
3038 Assert(VMMRZCallRing3IsEnabled(pVCpu));
3039 HMSVM_ASSERT_PREEMPT_SAFE();
3040
3041 VMMRZCallRing3Disable(pVCpu);
3042 Assert(VMMR0IsLogFlushDisabled(pVCpu));
3043
3044 Log4(("hmR0SvmCallRing3Callback->hmR0SvmLongJmpToRing3\n"));
3045 int rc = hmR0SvmLongJmpToRing3(pVCpu);
3046 AssertRCReturn(rc, rc);
3047
3048 VMMRZCallRing3Enable(pVCpu);
3049 return VINF_SUCCESS;
3050}
3051
3052
3053/**
3054 * Take necessary actions before going back to ring-3.
3055 *
3056 * An action requires us to go back to ring-3. This function does the necessary
3057 * steps before we can safely return to ring-3. This is not the same as longjmps
3058 * to ring-3, this is voluntary.
3059 *
3060 * @returns VBox status code.
3061 * @param pVM The cross context VM structure.
3062 * @param pVCpu The cross context virtual CPU structure.
3063 * @param pCtx Pointer to the guest-CPU context.
3064 * @param rcExit The reason for exiting to ring-3. Can be
3065 * VINF_VMM_UNKNOWN_RING3_CALL.
3066 */
3067static int hmR0SvmExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rcExit)
3068{
3069 Assert(pVM);
3070 Assert(pVCpu);
3071 Assert(pCtx);
3072 HMSVM_ASSERT_PREEMPT_SAFE();
3073
3074 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3075 VMMRZCallRing3Disable(pVCpu);
3076 Log4(("hmR0SvmExitToRing3: VCPU[%u]: rcExit=%d LocalFF=%#RX32 GlobalFF=%#RX32\n", pVCpu->idCpu, rcExit,
3077 pVCpu->fLocalForcedActions, pVM->fGlobalForcedActions));
3078
3079 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3080 if (pVCpu->hm.s.Event.fPending)
3081 {
3082 hmR0SvmPendingEventToTrpmTrap(pVCpu);
3083 Assert(!pVCpu->hm.s.Event.fPending);
3084 }
3085
3086 /* Sync. the necessary state for going back to ring-3. */
3087 hmR0SvmLeaveSession(pVCpu);
3088 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3089
3090 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3091 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3092 | CPUM_CHANGED_LDTR
3093 | CPUM_CHANGED_GDTR
3094 | CPUM_CHANGED_IDTR
3095 | CPUM_CHANGED_TR
3096 | CPUM_CHANGED_HIDDEN_SEL_REGS);
3097 if ( pVM->hm.s.fNestedPaging
3098 && CPUMIsGuestPagingEnabledEx(pCtx))
3099 {
3100 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3101 }
3102
3103 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
3104 if (rcExit != VINF_EM_RAW_INTERRUPT)
3105 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
3106
3107 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3108
3109 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
3110 VMMRZCallRing3RemoveNotification(pVCpu);
3111 VMMRZCallRing3Enable(pVCpu);
3112
3113 /*
3114 * If we're emulating an instruction, we shouldn't have any TRPM traps pending
3115 * and if we're injecting an event we should have a TRPM trap pending.
3116 */
3117 AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu),
3118 pVCpu->hm.s.u32HMError = rcExit,
3119 VERR_SVM_IPE_5);
3120 AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu),
3121 pVCpu->hm.s.u32HMError = rcExit,
3122 VERR_SVM_IPE_4);
3123
3124 return rcExit;
3125}
3126
3127
3128/**
3129 * Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3130 * intercepts.
3131 *
3132 * @param pVM The cross context VM structure.
3133 * @param pVCpu The cross context virtual CPU structure.
3134 * @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
3135 * @param pVmcb Pointer to the VM control block.
3136 *
3137 * @remarks No-long-jump zone!!!
3138 */
3139static void hmR0SvmUpdateTscOffsetting(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
3140{
3141 /*
3142 * Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3143 * and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3144 * RDTSC/RDTSCP as well.
3145 */
3146 bool fParavirtTsc;
3147 uint64_t uTscOffset;
3148 bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
3149
3150 bool fIntercept;
3151 if (fCanUseRealTsc)
3152 fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP);
3153 else
3154 {
3155 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP);
3156 fIntercept = true;
3157 }
3158
3159 if (!fIntercept)
3160 {
3161 /* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3162 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3163 uTscOffset = HMSvmNstGstApplyTscOffset(pVCpu, uTscOffset);
3164
3165 /* Update the TSC offset in the VMCB and the relevant clean bits. */
3166 pVmcb->ctrl.u64TSCOffset = uTscOffset;
3167 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3168
3169 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
3170 }
3171 else
3172 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
3173
3174 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3175 information before every VM-entry, hence we have nothing to do here at the moment. */
3176 if (fParavirtTsc)
3177 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3178}
3179
3180
3181/**
3182 * Sets an event as a pending event to be injected into the guest.
3183 *
3184 * @param pVCpu The cross context virtual CPU structure.
3185 * @param pEvent Pointer to the SVM event.
3186 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3187 * page-fault.
3188 *
3189 * @remarks Statistics counter assumes this is a guest event being reflected to
3190 * the guest i.e. 'StatInjectPendingReflect' is incremented always.
3191 */
3192DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3193{
3194 Assert(!pVCpu->hm.s.Event.fPending);
3195 Assert(pEvent->n.u1Valid);
3196
3197 pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3198 pVCpu->hm.s.Event.fPending = true;
3199 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3200
3201 Log4(("hmR0SvmSetPendingEvent: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
3202 pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3203}
3204
3205
3206/**
3207 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3208 *
3209 * @param pVCpu The cross context virtual CPU structure.
3210 */
3211DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
3212{
3213 SVMEVENT Event;
3214 Event.u = 0;
3215 Event.n.u1Valid = 1;
3216 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3217 Event.n.u8Vector = X86_XCPT_UD;
3218 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3219}
3220
3221
3222/**
3223 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
3224 *
3225 * @param pVCpu The cross context virtual CPU structure.
3226 */
3227DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
3228{
3229 SVMEVENT Event;
3230 Event.u = 0;
3231 Event.n.u1Valid = 1;
3232 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3233 Event.n.u8Vector = X86_XCPT_DB;
3234 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3235}
3236
3237
3238/**
3239 * Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3240 *
3241 * @param pVCpu The cross context virtual CPU structure.
3242 * @param pCtx Pointer to the guest-CPU context.
3243 * @param u32ErrCode The error-code for the page-fault.
3244 * @param uFaultAddress The page fault address (CR2).
3245 *
3246 * @remarks This updates the guest CR2 with @a uFaultAddress!
3247 */
3248DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3249{
3250 SVMEVENT Event;
3251 Event.u = 0;
3252 Event.n.u1Valid = 1;
3253 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3254 Event.n.u8Vector = X86_XCPT_PF;
3255 Event.n.u1ErrorCodeValid = 1;
3256 Event.n.u32ErrorCode = u32ErrCode;
3257
3258 /* Update CR2 of the guest. */
3259 if (pCtx->cr2 != uFaultAddress)
3260 {
3261 pCtx->cr2 = uFaultAddress;
3262 /* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3263 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR2);
3264 }
3265
3266 hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3267}
3268
3269
3270/**
3271 * Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3272 *
3273 * @param pVCpu The cross context virtual CPU structure.
3274 */
3275DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
3276{
3277 SVMEVENT Event;
3278 Event.u = 0;
3279 Event.n.u1Valid = 1;
3280 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3281 Event.n.u8Vector = X86_XCPT_MF;
3282 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3283}
3284
3285
3286/**
3287 * Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3288 *
3289 * @param pVCpu The cross context virtual CPU structure.
3290 */
3291DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
3292{
3293 SVMEVENT Event;
3294 Event.u = 0;
3295 Event.n.u1Valid = 1;
3296 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3297 Event.n.u8Vector = X86_XCPT_DF;
3298 Event.n.u1ErrorCodeValid = 1;
3299 Event.n.u32ErrorCode = 0;
3300 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3301}
3302
3303
3304/**
3305 * Injects an event into the guest upon VMRUN by updating the relevant field
3306 * in the VMCB.
3307 *
3308 * @param pVCpu The cross context virtual CPU structure.
3309 * @param pVmcb Pointer to the guest VM control block.
3310 * @param pCtx Pointer to the guest-CPU context.
3311 * @param pEvent Pointer to the event.
3312 *
3313 * @remarks No-long-jump zone!!!
3314 * @remarks Requires CR0!
3315 */
3316DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx, PSVMEVENT pEvent)
3317{
3318 NOREF(pVCpu); NOREF(pCtx);
3319
3320 Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3321 pVmcb->ctrl.EventInject.u = pEvent->u;
3322 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3323
3324 Log4(("hmR0SvmInjectEventVmcb: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
3325 pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3326}
3327
3328
3329
3330/**
3331 * Converts any TRPM trap into a pending HM event. This is typically used when
3332 * entering from ring-3 (not longjmp returns).
3333 *
3334 * @param pVCpu The cross context virtual CPU structure.
3335 */
3336static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
3337{
3338 Assert(TRPMHasTrap(pVCpu));
3339 Assert(!pVCpu->hm.s.Event.fPending);
3340
3341 uint8_t uVector;
3342 TRPMEVENT enmTrpmEvent;
3343 RTGCUINT uErrCode;
3344 RTGCUINTPTR GCPtrFaultAddress;
3345 uint8_t cbInstr;
3346
3347 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
3348 AssertRC(rc);
3349
3350 SVMEVENT Event;
3351 Event.u = 0;
3352 Event.n.u1Valid = 1;
3353 Event.n.u8Vector = uVector;
3354
3355 /* Refer AMD spec. 15.20 "Event Injection" for the format. */
3356 if (enmTrpmEvent == TRPM_TRAP)
3357 {
3358 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3359 switch (uVector)
3360 {
3361 case X86_XCPT_NMI:
3362 {
3363 Event.n.u3Type = SVM_EVENT_NMI;
3364 break;
3365 }
3366
3367 case X86_XCPT_PF:
3368 case X86_XCPT_DF:
3369 case X86_XCPT_TS:
3370 case X86_XCPT_NP:
3371 case X86_XCPT_SS:
3372 case X86_XCPT_GP:
3373 case X86_XCPT_AC:
3374 {
3375 Event.n.u1ErrorCodeValid = 1;
3376 Event.n.u32ErrorCode = uErrCode;
3377 break;
3378 }
3379 }
3380 }
3381 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3382 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3383 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3384 Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3385 else
3386 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3387
3388 rc = TRPMResetTrap(pVCpu);
3389 AssertRC(rc);
3390
3391 Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3392 !!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3393
3394 hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3395}
3396
3397
3398/**
3399 * Converts any pending SVM event into a TRPM trap. Typically used when leaving
3400 * AMD-V to execute any instruction.
3401 *
3402 * @param pVCpu The cross context virtual CPU structure.
3403 */
3404static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
3405{
3406 Assert(pVCpu->hm.s.Event.fPending);
3407 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3408
3409 SVMEVENT Event;
3410 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3411
3412 uint8_t uVector = Event.n.u8Vector;
3413 uint8_t uVectorType = Event.n.u3Type;
3414 TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event);
3415
3416 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, uVectorType));
3417
3418 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3419 AssertRC(rc);
3420
3421 if (Event.n.u1ErrorCodeValid)
3422 TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3423
3424 if ( uVectorType == SVM_EVENT_EXCEPTION
3425 && uVector == X86_XCPT_PF)
3426 {
3427 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3428 Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3429 }
3430 else if (uVectorType == SVM_EVENT_SOFTWARE_INT)
3431 {
3432 AssertMsg( uVectorType == SVM_EVENT_SOFTWARE_INT
3433 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
3434 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
3435 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3436 }
3437 pVCpu->hm.s.Event.fPending = false;
3438}
3439
3440
3441/**
3442 * Checks if the guest (or nested-guest) has an interrupt shadow active right
3443 * now.
3444 *
3445 * @returns @c true if the interrupt shadow is active, @c false otherwise.
3446 * @param pVCpu The cross context virtual CPU structure.
3447 * @param pCtx Pointer to the guest-CPU context.
3448 *
3449 * @remarks No-long-jump zone!!!
3450 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3451 */
3452DECLINLINE(bool) hmR0SvmIsIntrShadowActive(PVMCPU pVCpu, PCPUMCTX pCtx)
3453{
3454 /*
3455 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3456 * inhibit interrupts or clear any existing interrupt-inhibition.
3457 */
3458 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3459 {
3460 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3461 {
3462 /*
3463 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3464 * AMD-V, the flag's condition to be cleared is met and thus the cleared state is correct.
3465 */
3466 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3467 return false;
3468 }
3469 return true;
3470 }
3471 return false;
3472}
3473
3474
3475/**
3476 * Sets the virtual interrupt intercept control in the VMCB.
3477 *
3478 * @param pVCpu The cross context virtual CPU structure.
3479 * @param pVmcb Pointer to the VM control block.
3480 * @param pCtx Pointer to the guest-CPU context.
3481 */
3482DECLINLINE(void) hmR0SvmSetIntWindowExiting(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
3483{
3484 /*
3485 * When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when
3486 * the guest is ready to accept interrupts. At #VMEXIT, we then get the interrupt
3487 * from the APIC (updating ISR at the right time) and inject the interrupt.
3488 *
3489 * With AVIC is supported, we could make use of the asynchronously delivery without
3490 * #VMEXIT and we would be passing the AVIC page to SVM.
3491 *
3492 * In AMD-V, an interrupt window is achieved using a combination of
3493 * V_IRQ (an interrupt is pending), V_IGN_TPR (ignore TPR priorities) and the
3494 * VINTR intercept all being set.
3495 */
3496#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3497 /*
3498 * Currently we don't overlay interupt windows and if there's any V_IRQ pending
3499 * in the nested-guest VMCB, we avoid setting up any interrupt window on behalf
3500 * of the outer guest.
3501 */
3502 /** @todo Does this mean we end up prioritizing virtual interrupt
3503 * delivery/window over a physical interrupt (from the outer guest)
3504 * might be pending? */
3505 bool const fEnableIntWindow = !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
3506 if (!fEnableIntWindow)
3507 {
3508 Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); RT_NOREF(pCtx);
3509 Log4(("Nested-guest V_IRQ already pending\n"));
3510 }
3511#else
3512 RT_NOREF2(pVCpu, pCtx);
3513 bool const fEnableIntWindow = true;
3514#endif
3515 if (fEnableIntWindow)
3516 {
3517 Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3518 pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3519 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3520 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3521 Log4(("Set VINTR intercept\n"));
3522 }
3523}
3524
3525
3526/**
3527 * Clears the virtual interrupt intercept control in the VMCB as
3528 * we are figured the guest is unable process any interrupts
3529 * at this point of time.
3530 *
3531 * @param pVCpu The cross context virtual CPU structure.
3532 * @param pVmcb Pointer to the VM control block.
3533 * @param pCtx Pointer to the guest-CPU context.
3534 */
3535DECLINLINE(void) hmR0SvmClearIntWindowExiting(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
3536{
3537 PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3538 if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3539 || (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3540 {
3541 pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3542 pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3543 hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3544 Log4(("Cleared VINTR intercept\n"));
3545 }
3546}
3547
3548#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3549/**
3550 * Evaluates the event to be delivered to the nested-guest and sets it as the
3551 * pending event.
3552 *
3553 * @returns VBox strict status code.
3554 * @param pVCpu The cross context virtual CPU structure.
3555 * @param pCtx Pointer to the guest-CPU context.
3556 */
3557static VBOXSTRICTRC hmR0SvmEvaluatePendingEventNested(PVMCPU pVCpu, PCPUMCTX pCtx)
3558{
3559 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
3560
3561 Assert(!pVCpu->hm.s.Event.fPending);
3562 Assert(pCtx->hwvirt.fGif);
3563 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
3564 Assert(pVmcb);
3565
3566 bool const fVirtualGif = CPUMGetSvmNstGstVGif(pCtx);
3567 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3568 bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3569
3570 Log4Func(("fVirtualGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool Intr. pending=%RTbool NMI pending=%RTbool\n",
3571 fVirtualGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC),
3572 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3573
3574 /** @todo SMI. SMIs take priority over NMIs. */
3575
3576 /*
3577 * Check if the guest can receive NMIs.
3578 * Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3579 * NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3580 */
3581 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3582 && !fBlockNmi)
3583 {
3584 if ( fVirtualGif
3585 && !fIntShadow)
3586 {
3587 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3588 {
3589 Log4(("Intercepting NMI -> #VMEXIT\n"));
3590 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3591 }
3592
3593 Log4(("Setting NMI pending for injection\n"));
3594 SVMEVENT Event;
3595 Event.u = 0;
3596 Event.n.u1Valid = 1;
3597 Event.n.u8Vector = X86_XCPT_NMI;
3598 Event.n.u3Type = SVM_EVENT_NMI;
3599 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3600 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3601 }
3602 else if (!fVirtualGif)
3603 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3604 else
3605 hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3606 }
3607 /*
3608 * Check if the nested-guest can receive external interrupts (generated by
3609 * the guest's PIC/APIC).
3610 *
3611 * External intercepts, NMI, SMI etc. from the physical CPU are -always- intercepted
3612 * when executing using hardware-assisted SVM, see HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS.
3613 *
3614 * External interrupts that are generated for the outer guest may be intercepted
3615 * depending on how the nested-guest VMCB was programmed by guest software.
3616 *
3617 * Physical interrupts always take priority over virtual interrupts,
3618 * see AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3619 */
3620 else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
3621 && !pVCpu->hm.s.fSingleInstruction)
3622 {
3623 if ( fVirtualGif
3624 && !fIntShadow
3625 && CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx))
3626 {
3627 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3628 {
3629 Log4(("Intercepting INTR -> #VMEXIT\n"));
3630 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3631 }
3632
3633 uint8_t u8Interrupt;
3634 int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3635 if (RT_SUCCESS(rc))
3636 {
3637 Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3638 SVMEVENT Event;
3639 Event.u = 0;
3640 Event.n.u1Valid = 1;
3641 Event.n.u8Vector = u8Interrupt;
3642 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3643 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3644 }
3645 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3646 {
3647 /*
3648 * AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3649 * updated eventually when the TPR is written by the guest.
3650 */
3651 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3652 }
3653 else
3654 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3655 }
3656 else if (!fVirtualGif)
3657 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3658 else
3659 hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3660 }
3661
3662 return VINF_SUCCESS;
3663}
3664#endif
3665
3666/**
3667 * Evaluates the event to be delivered to the guest and sets it as the pending
3668 * event.
3669 *
3670 * @param pVCpu The cross context virtual CPU structure.
3671 * @param pCtx Pointer to the guest-CPU context.
3672 */
3673static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
3674{
3675 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3676 Assert(!pVCpu->hm.s.Event.fPending);
3677 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
3678 Assert(pVmcb);
3679
3680#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3681 bool const fGif = pCtx->hwvirt.fGif;
3682#else
3683 bool const fGif = true;
3684#endif
3685 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3686 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3687 bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3688
3689 Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fBlockInt=%RTbool fIntShadow=%RTbool Intr. pending=%RTbool NMI pending=%RTbool\n",
3690 fGif, fBlockNmi, fBlockInt, fIntShadow,
3691 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC),
3692 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3693
3694 /** @todo SMI. SMIs take priority over NMIs. */
3695
3696 /*
3697 * Check if the guest can receive NMIs.
3698 * Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3699 * NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3700 */
3701 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3702 && !fBlockNmi)
3703 {
3704 if ( fGif
3705 && !fIntShadow)
3706 {
3707 Log4(("Setting NMI pending for injection\n"));
3708 SVMEVENT Event;
3709 Event.u = 0;
3710 Event.n.u1Valid = 1;
3711 Event.n.u8Vector = X86_XCPT_NMI;
3712 Event.n.u3Type = SVM_EVENT_NMI;
3713 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3714 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3715 }
3716 else if (!fGif)
3717 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3718 else
3719 hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3720 }
3721 /*
3722 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
3723 * a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
3724 */
3725 else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
3726 && !pVCpu->hm.s.fSingleInstruction)
3727 {
3728 if ( fGif
3729 && !fBlockInt
3730 && !fIntShadow)
3731 {
3732 uint8_t u8Interrupt;
3733 int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3734 if (RT_SUCCESS(rc))
3735 {
3736 Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3737 SVMEVENT Event;
3738 Event.u = 0;
3739 Event.n.u1Valid = 1;
3740 Event.n.u8Vector = u8Interrupt;
3741 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3742 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3743 }
3744 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3745 {
3746 /*
3747 * AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3748 * updated eventually when the TPR is written by the guest.
3749 */
3750 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3751 }
3752 else
3753 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3754 }
3755 else if (!fGif)
3756 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3757 else
3758 hmR0SvmSetIntWindowExiting(pVCpu, pVmcb, pCtx);
3759 }
3760}
3761
3762
3763/**
3764 * Injects any pending events into the guest (or nested-guest).
3765 *
3766 * @param pVCpu The cross context virtual CPU structure.
3767 * @param pCtx Pointer to the guest-CPU context.
3768 * @param pVmcb Pointer to the VM control block.
3769 *
3770 * @remarks Must only be called when we are guaranteed to enter
3771 * hardware-assisted SVM execution and not return to ring-3
3772 * prematurely.
3773 */
3774static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
3775{
3776 Assert(!TRPMHasTrap(pVCpu));
3777 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3778
3779 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3780#ifdef VBOX_STRICT
3781 bool const fGif = pCtx->hwvirt.fGif;
3782 bool fAllowInt = fGif;
3783 if (fGif)
3784 {
3785 /*
3786 * For nested-guests we have no way to determine if we're injecting a physical or virtual
3787 * interrupt at this point. Hence the partial verification below.
3788 */
3789 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3790 fAllowInt = CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx) || CPUMCanSvmNstGstTakeVirtIntr(pVCpu, pCtx);
3791 else
3792 fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3793 }
3794#endif
3795
3796 if (pVCpu->hm.s.Event.fPending)
3797 {
3798 SVMEVENT Event;
3799 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3800 Assert(Event.n.u1Valid);
3801
3802 /*
3803 * Validate event injection pre-conditions.
3804 */
3805 if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3806 {
3807 Assert(fAllowInt);
3808 Assert(!fIntShadow);
3809 }
3810 else if (Event.n.u3Type == SVM_EVENT_NMI)
3811 {
3812 Assert(fGif);
3813 Assert(!fIntShadow);
3814 }
3815
3816 /*
3817 * Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We do this only
3818 * when we are surely going to inject the NMI as otherwise if we return to ring-3 prematurely we
3819 * could leave NMIs blocked indefinitely upon re-entry into SVM R0.
3820 *
3821 * With VT-x, this is handled by the Guest interruptibility information VMCS field which will set
3822 * the VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
3823 */
3824 if ( Event.n.u3Type == SVM_EVENT_NMI
3825 && Event.n.u8Vector == X86_XCPT_NMI
3826 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3827 {
3828 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3829 }
3830
3831 /*
3832 * Inject it (update VMCB for injection by the hardware).
3833 */
3834 Log4(("Injecting pending HM event\n"));
3835 hmR0SvmInjectEventVmcb(pVCpu, pVmcb, pCtx, &Event);
3836 pVCpu->hm.s.Event.fPending = false;
3837
3838 if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3839 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3840 else
3841 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3842 }
3843 else
3844 Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3845
3846 /*
3847 * We could have injected an NMI through IEM and continue guest execution using
3848 * hardware-assisted SVM. In which case, we would not have any events pending (above)
3849 * but we still need to intercept IRET in order to eventually clear NMI inhibition.
3850 */
3851 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3852 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3853
3854 /*
3855 * Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3856 *
3857 * For nested-guests: We need to update it too for the scenario where IEM executes
3858 * the nested-guest but execution later continues here with an interrupt shadow active.
3859 */
3860 pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3861}
3862
3863
3864/**
3865 * Reports world-switch error and dumps some useful debug info.
3866 *
3867 * @param pVM The cross context VM structure.
3868 * @param pVCpu The cross context virtual CPU structure.
3869 * @param rcVMRun The return code from VMRUN (or
3870 * VERR_SVM_INVALID_GUEST_STATE for invalid
3871 * guest-state).
3872 * @param pCtx Pointer to the guest-CPU context.
3873 */
3874static void hmR0SvmReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx)
3875{
3876 NOREF(pCtx);
3877 HMSVM_ASSERT_PREEMPT_SAFE();
3878 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3879 PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3880
3881 if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3882 {
3883 hmR0DumpRegs(pVM, pVCpu, pCtx); NOREF(pVM);
3884#ifdef VBOX_STRICT
3885 Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3886 Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3887 Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3888 Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3889 Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3890 Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3891 Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3892 Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3893 Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3894 Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3895
3896 Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3897 Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3898 Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3899
3900 Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3901 Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3902 Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3903 Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3904 Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3905 Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3906 Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3907 Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3908 Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3909 Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3910 Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3911 Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3912
3913 Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3914 Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3915 Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3916 Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3917 Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3918 Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3919 Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3920 Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3921 Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3922 Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3923 Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3924 Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3925 Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3926 Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3927 Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3928 Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3929 Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3930 Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3931 Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3932 Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3933
3934 Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3935
3936 Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3937 Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3938
3939 Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3940 Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3941 Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3942 Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3943 Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3944 Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3945 Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3946 Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3947 Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3948 Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3949 Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3950 Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3951 Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3952 Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3953 Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3954 Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3955 Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3956 Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3957 Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3958 Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3959
3960 Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3961 Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3962
3963 Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3964 Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3965 Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3966 Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3967
3968 Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3969 Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3970
3971 Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3972 Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3973 Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3974 Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3975
3976 Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3977 Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3978 Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3979 Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3980 Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3981 Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3982 Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3983
3984 Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3985 Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3986 Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3987 Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3988
3989 Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3990 Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3991 Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3992
3993 Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3994 Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3995 Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3996 Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3997 Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3998 Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3999 Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
4000 Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
4001 Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
4002 Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
4003 Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
4004 Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
4005#endif /* VBOX_STRICT */
4006 }
4007 else
4008 Log4(("hmR0SvmReportWorldSwitchError: rcVMRun=%d\n", rcVMRun));
4009
4010 NOREF(pVmcb);
4011}
4012
4013
4014/**
4015 * Check per-VM and per-VCPU force flag actions that require us to go back to
4016 * ring-3 for one reason or another.
4017 *
4018 * @returns VBox status code (information status code included).
4019 * @retval VINF_SUCCESS if we don't have any actions that require going back to
4020 * ring-3.
4021 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
4022 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
4023 * interrupts)
4024 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
4025 * all EMTs to be in ring-3.
4026 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
4027 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
4028 * to the EM loop.
4029 *
4030 * @param pVM The cross context VM structure.
4031 * @param pVCpu The cross context virtual CPU structure.
4032 * @param pCtx Pointer to the guest-CPU context.
4033 */
4034static int hmR0SvmCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4035{
4036 Assert(VMMRZCallRing3IsEnabled(pVCpu));
4037
4038 /* On AMD-V we don't need to update CR3, PAE PDPES lazily. See hmR0SvmSaveGuestState(). */
4039 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
4040 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
4041
4042 /* Update pending interrupts into the APIC's IRR. */
4043 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
4044 APICUpdatePendingInterrupts(pVCpu);
4045
4046 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
4047 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
4048 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
4049 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
4050 {
4051 /* Pending PGM C3 sync. */
4052 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
4053 {
4054 int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
4055 if (rc != VINF_SUCCESS)
4056 {
4057 Log4(("hmR0SvmCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
4058 return rc;
4059 }
4060 }
4061
4062 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4063 /* -XXX- what was that about single stepping? */
4064 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
4065 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4066 {
4067 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4068 int rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
4069 Log4(("hmR0SvmCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4070 return rc;
4071 }
4072
4073 /* Pending VM request packets, such as hardware interrupts. */
4074 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
4075 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
4076 {
4077 Log4(("hmR0SvmCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
4078 return VINF_EM_PENDING_REQUEST;
4079 }
4080
4081 /* Pending PGM pool flushes. */
4082 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4083 {
4084 Log4(("hmR0SvmCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
4085 return VINF_PGM_POOL_FLUSH_PENDING;
4086 }
4087
4088 /* Pending DMA requests. */
4089 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
4090 {
4091 Log4(("hmR0SvmCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
4092 return VINF_EM_RAW_TO_R3;
4093 }
4094 }
4095
4096 return VINF_SUCCESS;
4097}
4098
4099
4100#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4101/**
4102 * Does the preparations before executing nested-guest code in AMD-V.
4103 *
4104 * @returns VBox status code (informational status codes included).
4105 * @retval VINF_SUCCESS if we can proceed with running the guest.
4106 * @retval VINF_* scheduling changes, we have to go back to ring-3.
4107 *
4108 * @param pVM The cross context VM structure.
4109 * @param pVCpu The cross context virtual CPU structure.
4110 * @param pCtx Pointer to the nested-guest-CPU context.
4111 * @param pSvmTransient Pointer to the SVM transient structure.
4112 *
4113 * @remarks Same caveats regarding longjumps as hmR0SvmPreRunGuest applies.
4114 * @sa hmR0SvmPreRunGuest.
4115 */
4116static int hmR0SvmPreRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4117{
4118 HMSVM_ASSERT_PREEMPT_SAFE();
4119 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4120
4121#ifdef VBOX_WITH_NESTED_HWVIRT_SVM_ONLY_IN_IEM
4122 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4123 return VINF_EM_RESCHEDULE_REM;
4124#endif
4125
4126 /* Check force flag actions that might require us to go back to ring-3. */
4127 int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
4128 if (rc != VINF_SUCCESS)
4129 return rc;
4130
4131 if (TRPMHasTrap(pVCpu))
4132 hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4133 else if (!pVCpu->hm.s.Event.fPending)
4134 {
4135 VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEventNested(pVCpu, pCtx);
4136 if ( rcStrict != VINF_SUCCESS
4137 || !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4138 return VBOXSTRICTRC_VAL(rcStrict);
4139 }
4140
4141 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4142
4143 /*
4144 * On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4145 * Just do it in software, see @bugref{8411}.
4146 * NB: If we could continue a task switch exit we wouldn't need to do this.
4147 */
4148 if (RT_UNLIKELY( !pVM->hm.s.svm.u32Features
4149 && pVCpu->hm.s.Event.fPending
4150 && SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4151 {
4152 return VINF_EM_RAW_INJECT_TRPM_EVENT;
4153 }
4154
4155#ifdef HMSVM_SYNC_FULL_NESTED_GUEST_STATE
4156 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
4157#endif
4158
4159 /*
4160 * Load the nested-guest state.
4161 */
4162 rc = hmR0SvmLoadGuestStateNested(pVCpu, pCtx);
4163 AssertRCReturn(rc, rc);
4164 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull); /** @todo Get new STAM counter for this? */
4165
4166 /* Ensure we've cached (and hopefully modified) the VMCB for execution using hardware-assisted SVM. */
4167 Assert(pCtx->hwvirt.svm.fHMCachedVmcb);
4168
4169 /*
4170 * No longjmps to ring-3 from this point on!!!
4171 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4172 * This also disables flushing of the R0-logger instance (if any).
4173 */
4174 VMMRZCallRing3Disable(pVCpu);
4175
4176 /*
4177 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
4178 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
4179 *
4180 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
4181 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
4182 *
4183 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
4184 * executing guest code.
4185 */
4186 pSvmTransient->fEFlags = ASMIntDisableFlags();
4187 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
4188 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4189 {
4190 ASMSetFlags(pSvmTransient->fEFlags);
4191 VMMRZCallRing3Enable(pVCpu);
4192 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4193 return VINF_EM_RAW_TO_R3;
4194 }
4195 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4196 {
4197 ASMSetFlags(pSvmTransient->fEFlags);
4198 VMMRZCallRing3Enable(pVCpu);
4199 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4200 return VINF_EM_RAW_INTERRUPT;
4201 }
4202 return VINF_SUCCESS;
4203}
4204#endif
4205
4206
4207/**
4208 * Does the preparations before executing guest code in AMD-V.
4209 *
4210 * This may cause longjmps to ring-3 and may even result in rescheduling to the
4211 * recompiler. We must be cautious what we do here regarding committing
4212 * guest-state information into the VMCB assuming we assuredly execute the guest
4213 * in AMD-V. If we fall back to the recompiler after updating the VMCB and
4214 * clearing the common-state (TRPM/forceflags), we must undo those changes so
4215 * that the recompiler can (and should) use them when it resumes guest
4216 * execution. Otherwise such operations must be done when we can no longer
4217 * exit to ring-3.
4218 *
4219 * @returns VBox status code (informational status codes included).
4220 * @retval VINF_SUCCESS if we can proceed with running the guest.
4221 * @retval VINF_* scheduling changes, we have to go back to ring-3.
4222 *
4223 * @param pVM The cross context VM structure.
4224 * @param pVCpu The cross context virtual CPU structure.
4225 * @param pCtx Pointer to the guest-CPU context.
4226 * @param pSvmTransient Pointer to the SVM transient structure.
4227 */
4228static int hmR0SvmPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4229{
4230 HMSVM_ASSERT_PREEMPT_SAFE();
4231 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
4232
4233 /* Check force flag actions that might require us to go back to ring-3. */
4234 int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
4235 if (rc != VINF_SUCCESS)
4236 return rc;
4237
4238 if (TRPMHasTrap(pVCpu))
4239 hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4240 else if (!pVCpu->hm.s.Event.fPending)
4241 hmR0SvmEvaluatePendingEvent(pVCpu, pCtx);
4242
4243 /*
4244 * On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4245 * Just do it in software, see @bugref{8411}.
4246 * NB: If we could continue a task switch exit we wouldn't need to do this.
4247 */
4248 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending && (((pVCpu->hm.s.Event.u64IntInfo >> 8) & 7) == SVM_EVENT_NMI)))
4249 if (RT_UNLIKELY(!pVM->hm.s.svm.u32Features))
4250 return VINF_EM_RAW_INJECT_TRPM_EVENT;
4251
4252#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4253 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
4254#endif
4255
4256 /* Load the guest bits that are not shared with the host in any way since we can longjmp or get preempted. */
4257 rc = hmR0SvmLoadGuestState(pVM, pVCpu, pCtx);
4258 AssertRCReturn(rc, rc);
4259 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
4260
4261 /*
4262 * If we're not intercepting TPR changes in the guest, save the guest TPR before the world-switch
4263 * so we can update it on the way back if the guest changed the TPR.
4264 */
4265 if (pVCpu->hm.s.svm.fSyncVTpr)
4266 {
4267 if (pVM->hm.s.fTPRPatchingActive)
4268 pSvmTransient->u8GuestTpr = pCtx->msrLSTAR;
4269 else
4270 {
4271 PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4272 pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4273 }
4274 }
4275
4276 /*
4277 * No longjmps to ring-3 from this point on!!!
4278 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4279 * This also disables flushing of the R0-logger instance (if any).
4280 */
4281 VMMRZCallRing3Disable(pVCpu);
4282
4283 /*
4284 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
4285 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
4286 *
4287 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
4288 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
4289 *
4290 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
4291 * executing guest code.
4292 */
4293 pSvmTransient->fEFlags = ASMIntDisableFlags();
4294 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
4295 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4296 {
4297 ASMSetFlags(pSvmTransient->fEFlags);
4298 VMMRZCallRing3Enable(pVCpu);
4299 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4300 return VINF_EM_RAW_TO_R3;
4301 }
4302 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4303 {
4304 ASMSetFlags(pSvmTransient->fEFlags);
4305 VMMRZCallRing3Enable(pVCpu);
4306 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4307 return VINF_EM_RAW_INTERRUPT;
4308 }
4309
4310 return VINF_SUCCESS;
4311}
4312
4313
4314/**
4315 * Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4316 * doing so.
4317 *
4318 * This means there is no backing out to ring-3 or anywhere else at this point.
4319 *
4320 * @param pVCpu The cross context virtual CPU structure.
4321 * @param pCtx Pointer to the guest-CPU context.
4322 * @param pSvmTransient Pointer to the SVM transient structure.
4323 *
4324 * @remarks Called with preemption disabled.
4325 * @remarks No-long-jump zone!!!
4326 */
4327static void hmR0SvmPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4328{
4329 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4330 Assert(VMMR0IsLogFlushDisabled(pVCpu));
4331 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4332
4333 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4334 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4335
4336 PVM pVM = pVCpu->CTX_SUFF(pVM);
4337 PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4338
4339 hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcb);
4340
4341 if (!CPUMIsGuestFPUStateActive(pVCpu))
4342 {
4343 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4344 CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4345 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4346 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4347 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4348 }
4349
4350 /* Load the state shared between host and guest (FPU, debug). */
4351 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4352 hmR0SvmLoadSharedState(pVCpu, pVmcb, pCtx);
4353
4354 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4355 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4356
4357 PHMGLOBALCPUINFO pHostCpu = hmR0GetCurrentCpu();
4358 RTCPUID const idHostCpu = pHostCpu->idCpu;
4359 bool const fMigratedHostCpu = idHostCpu != pVCpu->hm.s.idLastCpu;
4360
4361 /* Setup TSC offsetting. */
4362 if ( pSvmTransient->fUpdateTscOffsetting
4363 || fMigratedHostCpu)
4364 {
4365 hmR0SvmUpdateTscOffsetting(pVM, pVCpu, pCtx, pVmcb);
4366 pSvmTransient->fUpdateTscOffsetting = false;
4367 }
4368
4369 /* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4370 if (fMigratedHostCpu)
4371 pVmcb->ctrl.u32VmcbCleanBits = 0;
4372
4373 /* Store status of the shared guest-host state at the time of VMRUN. */
4374#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4375 if (CPUMIsGuestInLongModeEx(pCtx))
4376 {
4377 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4378 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4379 }
4380 else
4381#endif
4382 {
4383 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4384 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4385 }
4386
4387 uint8_t *pbMsrBitmap;
4388 if (!pSvmTransient->fIsNestedGuest)
4389 pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
4390#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4391 else
4392 {
4393 hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu, pCtx);
4394
4395 /* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4396 pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4397 pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4398 }
4399#endif
4400
4401 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4402 /* Flush the appropriate tagged-TLB entries. */
4403 hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcb, pHostCpu);
4404 Assert(pVCpu->hm.s.idLastCpu == idHostCpu);
4405
4406 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4407
4408 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4409 to start executing. */
4410
4411 /*
4412 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4413 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4414 *
4415 * This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4416 */
4417 if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4418 && !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4419 {
4420 uint64_t const uGuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4421 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4422 if (uGuestTscAux != pVCpu->hm.s.u64HostTscAux)
4423 ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4424 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4425 pSvmTransient->fRestoreTscAuxMsr = true;
4426 }
4427 else
4428 {
4429 hmR0SvmSetMsrPermission(pCtx, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4430 pSvmTransient->fRestoreTscAuxMsr = false;
4431 }
4432 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4433
4434 /*
4435 * If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the
4436 * nested virtualization case, mark all state-bits as dirty indicating to the
4437 * CPU to re-load from VMCB.
4438 */
4439 bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4440 if (!fSupportsVmcbCleanBits)
4441 pVmcb->ctrl.u32VmcbCleanBits = 0;
4442}
4443
4444
4445/**
4446 * Wrapper for running the guest code in AMD-V.
4447 *
4448 * @returns VBox strict status code.
4449 * @param pVM The cross context VM structure.
4450 * @param pVCpu The cross context virtual CPU structure.
4451 * @param pCtx Pointer to the guest-CPU context.
4452 *
4453 * @remarks No-long-jump zone!!!
4454 */
4455DECLINLINE(int) hmR0SvmRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4456{
4457 /*
4458 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4459 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4460 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4461 */
4462#ifdef VBOX_WITH_KERNEL_USING_XMM
4463 return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4464 pVCpu->hm.s.svm.pfnVMRun);
4465#else
4466 return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4467#endif
4468}
4469
4470
4471#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4472/**
4473 * Undoes the TSC offset applied for an SVM nested-guest and returns the TSC
4474 * value for the guest.
4475 *
4476 * @returns The TSC offset after undoing any nested-guest TSC offset.
4477 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
4478 * @param uTicks The nested-guest TSC.
4479 *
4480 * @note If you make any changes to this function, please check if
4481 * hmR0SvmNstGstUndoTscOffset() needs adjusting.
4482 *
4483 * @sa HMSvmNstGstApplyTscOffset().
4484 */
4485DECLINLINE(uint64_t) hmR0SvmNstGstUndoTscOffset(PVMCPU pVCpu, PCPUMCTX pCtx, uint64_t uTicks)
4486{
4487 Assert(pCtx->hwvirt.svm.fHMCachedVmcb); RT_NOREF(pCtx);
4488 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
4489 return uTicks - pVmcbNstGstCache->u64TSCOffset;
4490}
4491
4492
4493/**
4494 * Wrapper for running the nested-guest code in AMD-V.
4495 *
4496 * @returns VBox strict status code.
4497 * @param pVM The cross context VM structure.
4498 * @param pVCpu The cross context virtual CPU structure.
4499 * @param pCtx Pointer to the guest-CPU context.
4500 *
4501 * @remarks No-long-jump zone!!!
4502 */
4503DECLINLINE(int) hmR0SvmRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4504{
4505 /*
4506 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4507 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4508 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4509 */
4510#ifdef VBOX_WITH_KERNEL_USING_XMM
4511 return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4512 pVCpu->hm.s.svm.pfnVMRun);
4513#else
4514 return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4515#endif
4516}
4517#endif
4518
4519/**
4520 * Performs some essential restoration of state after running guest (or
4521 * nested-guest) code in AMD-V.
4522 *
4523 * @param pVCpu The cross context virtual CPU structure.
4524 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4525 * out-of-sync. Make sure to update the required fields
4526 * before using them.
4527 * @param pSvmTransient Pointer to the SVM transient structure.
4528 * @param rcVMRun Return code of VMRUN.
4529 *
4530 * @remarks Called with interrupts disabled.
4531 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
4532 * unconditionally when it is safe to do so.
4533 */
4534static void hmR0SvmPostRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4535{
4536 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4537
4538 uint64_t const uHostTsc = ASMReadTSC(); /* Read the TSC as soon as possible. */
4539 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4540 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4541
4542 PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4543 PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4544
4545 /* TSC read must be done early for maximum accuracy. */
4546 if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4547 {
4548 if (!pSvmTransient->fIsNestedGuest)
4549 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVmcbCtrl->u64TSCOffset);
4550#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4551 else
4552 {
4553 /* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMSvmNstGstVmExitNotify(). */
4554 uint64_t const uGstTsc = hmR0SvmNstGstUndoTscOffset(pVCpu, pMixedCtx, uHostTsc + pVmcbCtrl->u64TSCOffset);
4555 TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4556 }
4557#endif
4558 }
4559
4560 if (pSvmTransient->fRestoreTscAuxMsr)
4561 {
4562 uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4563 CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4564 if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4565 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4566 }
4567
4568 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4569 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4570 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4571
4572 Assert(!(ASMGetFlags() & X86_EFL_IF));
4573 ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4574 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4575
4576 /* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4577 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4578 {
4579 Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4580 return;
4581 }
4582
4583 pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4584 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcbCtrl->u64ExitCode); /* Update the #VMEXIT history array. */
4585 pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4586 pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4587 pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4588
4589 hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcb); /* Save the guest state from the VMCB to the guest-CPU context. */
4590
4591 if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4592 && pVCpu->hm.s.svm.fSyncVTpr)
4593 {
4594 Assert(!pSvmTransient->fIsNestedGuest);
4595 /* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4596 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive
4597 && (pMixedCtx->msrLSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4598 {
4599 int rc = APICSetTpr(pVCpu, pMixedCtx->msrLSTAR & 0xff);
4600 AssertRC(rc);
4601 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
4602 }
4603 /* Sync TPR when we aren't intercepting CR8 writes. */
4604 else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4605 {
4606 int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4607 AssertRC(rc);
4608 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
4609 }
4610 }
4611}
4612
4613
4614/**
4615 * Runs the guest code using AMD-V.
4616 *
4617 * @returns VBox status code.
4618 * @param pVM The cross context VM structure.
4619 * @param pVCpu The cross context virtual CPU structure.
4620 * @param pCtx Pointer to the guest-CPU context.
4621 * @param pcLoops Pointer to the number of executed loops.
4622 */
4623static int hmR0SvmRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4624{
4625 uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4626 Assert(pcLoops);
4627 Assert(*pcLoops <= cMaxResumeLoops);
4628
4629 SVMTRANSIENT SvmTransient;
4630 RT_ZERO(SvmTransient);
4631 SvmTransient.fUpdateTscOffsetting = true;
4632 SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
4633
4634 int rc = VERR_INTERNAL_ERROR_5;
4635 for (;;)
4636 {
4637 Assert(!HMR0SuspendPending());
4638 HMSVM_ASSERT_CPU_SAFE();
4639
4640 /* Preparatory work for running guest code, this may force us to return
4641 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4642 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4643 rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4644 if (rc != VINF_SUCCESS)
4645 break;
4646
4647 /*
4648 * No longjmps to ring-3 from this point on!!!
4649 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4650 * This also disables flushing of the R0-logger instance (if any).
4651 */
4652 hmR0SvmPreRunGuestCommitted(pVCpu, pCtx, &SvmTransient);
4653 rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4654
4655 /* Restore any residual host-state and save any bits shared between host
4656 and guest into the guest-CPU state. Re-enables interrupts! */
4657 hmR0SvmPostRunGuest(pVCpu, pCtx, &SvmTransient, rc);
4658
4659 if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4660 || SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4661 {
4662 if (rc == VINF_SUCCESS)
4663 rc = VERR_SVM_INVALID_GUEST_STATE;
4664 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4665 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4666 break;
4667 }
4668
4669 /* Handle the #VMEXIT. */
4670 HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4671 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4672 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4673 rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4674 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4675 if (rc != VINF_SUCCESS)
4676 break;
4677 if (++(*pcLoops) >= cMaxResumeLoops)
4678 {
4679 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4680 rc = VINF_EM_RAW_INTERRUPT;
4681 break;
4682 }
4683 }
4684
4685 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4686 return rc;
4687}
4688
4689
4690/**
4691 * Runs the guest code using AMD-V in single step mode.
4692 *
4693 * @returns VBox status code.
4694 * @param pVM The cross context VM structure.
4695 * @param pVCpu The cross context virtual CPU structure.
4696 * @param pCtx Pointer to the guest-CPU context.
4697 * @param pcLoops Pointer to the number of executed loops.
4698 */
4699static int hmR0SvmRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4700{
4701 uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4702 Assert(pcLoops);
4703 Assert(*pcLoops <= cMaxResumeLoops);
4704
4705 SVMTRANSIENT SvmTransient;
4706 RT_ZERO(SvmTransient);
4707 SvmTransient.fUpdateTscOffsetting = true;
4708 SvmTransient.pVmcb = pVCpu->hm.s.svm.pVmcb;
4709
4710 uint16_t uCsStart = pCtx->cs.Sel;
4711 uint64_t uRipStart = pCtx->rip;
4712
4713 int rc = VERR_INTERNAL_ERROR_5;
4714 for (;;)
4715 {
4716 Assert(!HMR0SuspendPending());
4717 AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
4718 ("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
4719 (unsigned)RTMpCpuId(), *pcLoops));
4720
4721 /* Preparatory work for running guest code, this may force us to return
4722 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4723 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4724 rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4725 if (rc != VINF_SUCCESS)
4726 break;
4727
4728 /*
4729 * No longjmps to ring-3 from this point on!!!
4730 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4731 * This also disables flushing of the R0-logger instance (if any).
4732 */
4733 VMMRZCallRing3Disable(pVCpu);
4734 VMMRZCallRing3RemoveNotification(pVCpu);
4735 hmR0SvmPreRunGuestCommitted(pVCpu, pCtx, &SvmTransient);
4736
4737 rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4738
4739 /*
4740 * Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
4741 * This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
4742 */
4743 hmR0SvmPostRunGuest(pVCpu, pCtx, &SvmTransient, rc);
4744 if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4745 || SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4746 {
4747 if (rc == VINF_SUCCESS)
4748 rc = VERR_SVM_INVALID_GUEST_STATE;
4749 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4750 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4751 return rc;
4752 }
4753
4754 /* Handle the #VMEXIT. */
4755 HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4756 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4757 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4758 rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4759 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4760 if (rc != VINF_SUCCESS)
4761 break;
4762 if (++(*pcLoops) >= cMaxResumeLoops)
4763 {
4764 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4765 rc = VINF_EM_RAW_INTERRUPT;
4766 break;
4767 }
4768
4769 /*
4770 * Did the RIP change, if so, consider it a single step.
4771 * Otherwise, make sure one of the TFs gets set.
4772 */
4773 if ( pCtx->rip != uRipStart
4774 || pCtx->cs.Sel != uCsStart)
4775 {
4776 rc = VINF_EM_DBG_STEPPED;
4777 break;
4778 }
4779 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
4780 }
4781
4782 /*
4783 * Clear the X86_EFL_TF if necessary.
4784 */
4785 if (pVCpu->hm.s.fClearTrapFlag)
4786 {
4787 pVCpu->hm.s.fClearTrapFlag = false;
4788 pCtx->eflags.Bits.u1TF = 0;
4789 }
4790
4791 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4792 return rc;
4793}
4794
4795#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4796/**
4797 * Runs the nested-guest code using AMD-V.
4798 *
4799 * @returns VBox status code.
4800 * @param pVM The cross context VM structure.
4801 * @param pVCpu The cross context virtual CPU structure.
4802 * @param pCtx Pointer to the guest-CPU context.
4803 * @param pcLoops Pointer to the number of executed loops. If we're switching
4804 * from the guest-code execution loop to this nested-guest
4805 * execution loop pass the remainder value, else pass 0.
4806 */
4807static int hmR0SvmRunGuestCodeNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4808{
4809 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4810 Assert(pcLoops);
4811 Assert(*pcLoops <= pVM->hm.s.cMaxResumeLoops);
4812
4813 SVMTRANSIENT SvmTransient;
4814 RT_ZERO(SvmTransient);
4815 SvmTransient.fUpdateTscOffsetting = true;
4816 SvmTransient.pVmcb = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4817 SvmTransient.fIsNestedGuest = true;
4818
4819 int rc = VERR_INTERNAL_ERROR_4;
4820 for (;;)
4821 {
4822 Assert(!HMR0SuspendPending());
4823 HMSVM_ASSERT_CPU_SAFE();
4824
4825 /* Preparatory work for running nested-guest code, this may force us to return
4826 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4827 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4828 rc = hmR0SvmPreRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient);
4829 if ( rc != VINF_SUCCESS
4830 || !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4831 {
4832 break;
4833 }
4834
4835 /*
4836 * No longjmps to ring-3 from this point on!!!
4837 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4838 * This also disables flushing of the R0-logger instance (if any).
4839 */
4840 hmR0SvmPreRunGuestCommitted(pVCpu, pCtx, &SvmTransient);
4841
4842 rc = hmR0SvmRunGuestNested(pVM, pVCpu, pCtx);
4843
4844 /* Restore any residual host-state and save any bits shared between host
4845 and guest into the guest-CPU state. Re-enables interrupts! */
4846 hmR0SvmPostRunGuest(pVCpu, pCtx, &SvmTransient, rc);
4847
4848 if (RT_LIKELY( rc == VINF_SUCCESS
4849 && SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4850 { /* extremely likely */ }
4851 else
4852 {
4853 /* VMRUN failed, shouldn't really happen, Guru. */
4854 if (rc != VINF_SUCCESS)
4855 break;
4856
4857 /* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4858 AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4859 rc = VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0));
4860 break;
4861 }
4862
4863 /* Handle the #VMEXIT. */
4864 HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4865 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4866 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
4867 rc = hmR0SvmHandleExitNested(pVCpu, pCtx, &SvmTransient);
4868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4869 if ( rc != VINF_SUCCESS
4870 || !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4871 break;
4872 if (++(*pcLoops) >= pVM->hm.s.cMaxResumeLoops)
4873 {
4874 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4875 rc = VINF_EM_RAW_INTERRUPT;
4876 break;
4877 }
4878
4879 /** @todo handle single-stepping */
4880 }
4881
4882 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4883 return rc;
4884}
4885#endif
4886
4887
4888/**
4889 * Runs the guest code using AMD-V.
4890 *
4891 * @returns Strict VBox status code.
4892 * @param pVM The cross context VM structure.
4893 * @param pVCpu The cross context virtual CPU structure.
4894 * @param pCtx Pointer to the guest-CPU context.
4895 */
4896VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4897{
4898 Assert(VMMRZCallRing3IsEnabled(pVCpu));
4899 HMSVM_ASSERT_PREEMPT_SAFE();
4900 VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, pCtx);
4901
4902 uint32_t cLoops = 0;
4903 int rc;
4904#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4905 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4906#endif
4907 {
4908 if (!pVCpu->hm.s.fSingleInstruction)
4909 rc = hmR0SvmRunGuestCodeNormal(pVM, pVCpu, pCtx, &cLoops);
4910 else
4911 rc = hmR0SvmRunGuestCodeStep(pVM, pVCpu, pCtx, &cLoops);
4912 }
4913#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4914 else
4915 {
4916 rc = VINF_SVM_VMRUN;
4917 }
4918
4919 /* Re-check the nested-guest condition here as we may be transitioning from the normal
4920 execution loop into the nested-guest, hence this is not placed in the 'else' part above. */
4921 if (rc == VINF_SVM_VMRUN)
4922 {
4923 rc = hmR0SvmRunGuestCodeNested(pVM, pVCpu, pCtx, &cLoops);
4924 if (rc == VINF_SVM_VMEXIT)
4925 rc = VINF_SUCCESS;
4926 }
4927#endif
4928
4929 /* Fixup error codes. */
4930 if (rc == VERR_EM_INTERPRETER)
4931 rc = VINF_EM_RAW_EMULATE_INSTR;
4932 else if (rc == VINF_EM_RESET)
4933 rc = VINF_EM_TRIPLE_FAULT;
4934
4935 /* Prepare to return to ring-3. This will remove longjmp notifications. */
4936 rc = hmR0SvmExitToRing3(pVM, pVCpu, pCtx, rc);
4937 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
4938 return rc;
4939}
4940
4941
4942#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4943/**
4944 * Determines whether an IOIO intercept is active for the nested-guest or not.
4945 *
4946 * @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4947 * @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4948 */
4949static bool hmR0SvmIsIoInterceptActive(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4950{
4951 const uint16_t u16Port = pIoExitInfo->n.u16Port;
4952 const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4953 const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4954 const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4955 const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4956 const bool fRep = pIoExitInfo->n.u1Rep;
4957 const bool fStrIo = pIoExitInfo->n.u1Str;
4958
4959 return HMSvmIsIOInterceptActive(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4960 NULL /* pIoExitInfo */);
4961}
4962
4963
4964/**
4965 * Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4966 * SVM_EXIT_INVALID).
4967 *
4968 * @returns VBox status code (informational status codes included).
4969 * @param pVCpu The cross context virtual CPU structure.
4970 * @param pCtx Pointer to the guest-CPU context.
4971 * @param pSvmTransient Pointer to the SVM transient structure.
4972 */
4973static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4974{
4975 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4976 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4977 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4978
4979#define HM_SVM_VMEXIT_NESTED(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4980 VBOXSTRICTRC_TODO(IEMExecSvmVmexit(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2))
4981
4982 /*
4983 * For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected
4984 * by the nested-guest. If it isn't, it should be handled by the (outer) guest.
4985 */
4986 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4987 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4988 uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4989 uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4990 uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4991
4992 Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4993 switch (uExitCode)
4994 {
4995 case SVM_EXIT_CPUID:
4996 {
4997 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4998 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4999 return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
5000 }
5001
5002 case SVM_EXIT_RDTSC:
5003 {
5004 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
5005 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5006 return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
5007 }
5008
5009 case SVM_EXIT_RDTSCP:
5010 {
5011 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
5012 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5013 return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
5014 }
5015
5016 case SVM_EXIT_MONITOR:
5017 {
5018 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
5019 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5020 return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
5021 }
5022
5023 case SVM_EXIT_MWAIT:
5024 {
5025 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
5026 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5027 return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
5028 }
5029
5030 case SVM_EXIT_HLT:
5031 {
5032 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
5033 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5034 return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
5035 }
5036
5037 case SVM_EXIT_MSR:
5038 {
5039 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
5040 {
5041 uint32_t const idMsr = pCtx->ecx;
5042 uint16_t offMsrpm;
5043 uint8_t uMsrpmBit;
5044 int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
5045 if (RT_SUCCESS(rc))
5046 {
5047 Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
5048 Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
5049
5050 uint8_t const *pbMsrBitmap = (uint8_t const *)pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
5051 pbMsrBitmap += offMsrpm;
5052 bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
5053 bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
5054
5055 if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
5056 || (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
5057 {
5058 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5059 }
5060 }
5061 else
5062 {
5063 /*
5064 * MSRs not covered by the MSRPM automatically cause an #VMEXIT.
5065 * See AMD-V spec. "15.11 MSR Intercepts".
5066 */
5067 Assert(rc == VERR_OUT_OF_RANGE);
5068 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5069 }
5070 }
5071 return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
5072 }
5073
5074 case SVM_EXIT_IOIO:
5075 {
5076 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
5077 {
5078 void *pvIoBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvIoBitmap);
5079 SVMIOIOEXITINFO IoExitInfo;
5080 IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
5081 bool const fIntercept = hmR0SvmIsIoInterceptActive(pvIoBitmap, &IoExitInfo);
5082 if (fIntercept)
5083 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5084 }
5085 return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
5086 }
5087
5088 case SVM_EXIT_XCPT_PF:
5089 {
5090 PVM pVM = pVCpu->CTX_SUFF(pVM);
5091 if (pVM->hm.s.fNestedPaging)
5092 {
5093 uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
5094 uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
5095
5096 /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
5097 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
5098 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
5099
5100 /* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
5101 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
5102 return VINF_SUCCESS;
5103 }
5104 return hmR0SvmExitXcptPF(pVCpu, pCtx,pSvmTransient);
5105 }
5106
5107 case SVM_EXIT_XCPT_UD:
5108 {
5109 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
5110 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5111 hmR0SvmSetPendingXcptUD(pVCpu);
5112 return VINF_SUCCESS;
5113 }
5114
5115 case SVM_EXIT_XCPT_MF:
5116 {
5117 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
5118 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5119 return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
5120 }
5121
5122 case SVM_EXIT_XCPT_DB:
5123 {
5124 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
5125 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5126 return hmR0SvmNestedExitXcptDB(pVCpu, pCtx, pSvmTransient);
5127 }
5128
5129 case SVM_EXIT_XCPT_AC:
5130 {
5131 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5132 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5133 return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5134 }
5135
5136 case SVM_EXIT_XCPT_BP:
5137 {
5138 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5139 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5140 return hmR0SvmNestedExitXcptBP(pVCpu, pCtx, pSvmTransient);
5141 }
5142
5143 case SVM_EXIT_READ_CR0:
5144 case SVM_EXIT_READ_CR3:
5145 case SVM_EXIT_READ_CR4:
5146 {
5147 uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5148 if (HMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5149 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5150 return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5151 }
5152
5153 case SVM_EXIT_CR0_SEL_WRITE:
5154 {
5155 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5156 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5157 return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5158 }
5159
5160 case SVM_EXIT_WRITE_CR0:
5161 case SVM_EXIT_WRITE_CR3:
5162 case SVM_EXIT_WRITE_CR4:
5163 case SVM_EXIT_WRITE_CR8: /** @todo Shouldn't writes to CR8 go to V_TPR instead since we run with V_INTR_MASKING set? */
5164 {
5165 uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5166 Log4(("hmR0SvmHandleExitNested: Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5167
5168 if (HMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5169 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5170 return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5171 }
5172
5173 case SVM_EXIT_PAUSE:
5174 {
5175 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5176 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5177 return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5178 }
5179
5180 case SVM_EXIT_VINTR:
5181 {
5182 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5183 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5184 return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5185 }
5186
5187 case SVM_EXIT_INTR:
5188 case SVM_EXIT_NMI:
5189 case SVM_EXIT_XCPT_NMI: /* Shouldn't ever happen, SVM_EXIT_NMI is used instead. */
5190 case SVM_EXIT_SMI:
5191 {
5192 /*
5193 * We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5194 *
5195 * Although we don't intercept SMIs, the nested-guest might. Therefore, we
5196 * might get an SMI #VMEXIT here so simply ignore rather than causing a
5197 * corresponding nested-guest #VMEXIT.
5198 */
5199 return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5200 }
5201
5202 case SVM_EXIT_FERR_FREEZE:
5203 {
5204 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5205 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5206 return hmR0SvmExitFerrFreeze(pVCpu, pCtx, pSvmTransient);
5207 }
5208
5209 case SVM_EXIT_INVLPG:
5210 {
5211 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5212 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5213 return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5214 }
5215
5216 case SVM_EXIT_WBINVD:
5217 {
5218 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5219 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5220 return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5221 }
5222
5223 case SVM_EXIT_INVD:
5224 {
5225 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5226 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5227 return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5228 }
5229
5230 case SVM_EXIT_RDPMC:
5231 {
5232 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5233 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5234 return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5235 }
5236
5237 default:
5238 {
5239 switch (uExitCode)
5240 {
5241 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5242 case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5243 case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5244 case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5245 {
5246 uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5247 if (HMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5248 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5249 return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5250 }
5251
5252 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5253 case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5254 case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5255 case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5256 {
5257 uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5258 if (HMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5259 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5260 return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5261 }
5262
5263 case SVM_EXIT_XCPT_0: /* #DE */
5264 /* SVM_EXIT_XCPT_1: */ /* #DB - Handled above. */
5265 /* SVM_EXIT_XCPT_2: */ /* #NMI - Handled above. */
5266 /* SVM_EXIT_XCPT_3: */ /* #BP - Handled above. */
5267 case SVM_EXIT_XCPT_4: /* #OF */
5268 case SVM_EXIT_XCPT_5: /* #BR */
5269 /* SVM_EXIT_XCPT_6: */ /* #UD - Handled above. */
5270 case SVM_EXIT_XCPT_7: /* #NM */
5271 case SVM_EXIT_XCPT_8: /* #DF */
5272 case SVM_EXIT_XCPT_9: /* #CO_SEG_OVERRUN */
5273 case SVM_EXIT_XCPT_10: /* #TS */
5274 case SVM_EXIT_XCPT_11: /* #NP */
5275 case SVM_EXIT_XCPT_12: /* #SS */
5276 case SVM_EXIT_XCPT_13: /* #GP */
5277 /* SVM_EXIT_XCPT_14: */ /* #PF - Handled above. */
5278 case SVM_EXIT_XCPT_15: /* Reserved. */
5279 /* SVM_EXIT_XCPT_16: */ /* #MF - Handled above. */
5280 /* SVM_EXIT_XCPT_17: */ /* #AC - Handled above. */
5281 case SVM_EXIT_XCPT_18: /* #MC */
5282 case SVM_EXIT_XCPT_19: /* #XF */
5283 case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5284 case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5285 case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5286 {
5287 uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5288 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5289 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5290 return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5291 }
5292
5293 case SVM_EXIT_XSETBV:
5294 {
5295 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5296 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5297 return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5298 }
5299
5300 case SVM_EXIT_TASK_SWITCH:
5301 {
5302 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5303 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5304 return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5305 }
5306
5307 case SVM_EXIT_IRET:
5308 {
5309 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5310 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5311 return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5312 }
5313
5314 case SVM_EXIT_SHUTDOWN:
5315 {
5316 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5317 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5318 return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5319 }
5320
5321 case SVM_EXIT_VMMCALL:
5322 {
5323 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5324 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5325 return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5326 }
5327
5328 case SVM_EXIT_CLGI:
5329 {
5330 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5331 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5332 return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5333 }
5334
5335 case SVM_EXIT_STGI:
5336 {
5337 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5338 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5339 return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5340 }
5341
5342 case SVM_EXIT_VMLOAD:
5343 {
5344 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5345 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5346 return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5347 }
5348
5349 case SVM_EXIT_VMSAVE:
5350 {
5351 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5352 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5353 return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5354 }
5355
5356 case SVM_EXIT_INVLPGA:
5357 {
5358 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5359 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5360 return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5361 }
5362
5363 case SVM_EXIT_VMRUN:
5364 {
5365 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5366 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5367 return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5368 }
5369
5370 case SVM_EXIT_RSM:
5371 {
5372 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5373 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5374 hmR0SvmSetPendingXcptUD(pVCpu);
5375 return VINF_SUCCESS;
5376 }
5377
5378 case SVM_EXIT_SKINIT:
5379 {
5380 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5381 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5382 hmR0SvmSetPendingXcptUD(pVCpu);
5383 return VINF_SUCCESS;
5384 }
5385
5386 case SVM_EXIT_NPF:
5387 {
5388 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
5389 return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
5390 }
5391
5392 case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5393 return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5394
5395 default:
5396 {
5397 AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5398 pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5399 return VERR_SVM_UNKNOWN_EXIT;
5400 }
5401 }
5402 }
5403 }
5404 /* not reached */
5405
5406#undef HM_SVM_VMEXIT_NESTED
5407}
5408#endif
5409
5410
5411/**
5412 * Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5413 *
5414 * @returns VBox status code (informational status codes included).
5415 * @param pVCpu The cross context virtual CPU structure.
5416 * @param pCtx Pointer to the guest-CPU context.
5417 * @param pSvmTransient Pointer to the SVM transient structure.
5418 */
5419static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5420{
5421 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5422 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5423
5424 /*
5425 * The ordering of the case labels is based on most-frequently-occurring #VMEXITs for most guests under
5426 * normal workloads (for some definition of "normal").
5427 */
5428 uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5429 switch (uExitCode)
5430 {
5431 case SVM_EXIT_NPF:
5432 return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
5433
5434 case SVM_EXIT_IOIO:
5435 return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
5436
5437 case SVM_EXIT_RDTSC:
5438 return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
5439
5440 case SVM_EXIT_RDTSCP:
5441 return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
5442
5443 case SVM_EXIT_CPUID:
5444 return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
5445
5446 case SVM_EXIT_XCPT_14: /* X86_XCPT_PF */
5447 return hmR0SvmExitXcptPF(pVCpu, pCtx, pSvmTransient);
5448
5449 case SVM_EXIT_XCPT_6: /* X86_XCPT_UD */
5450 return hmR0SvmExitXcptUD(pVCpu, pCtx, pSvmTransient);
5451
5452 case SVM_EXIT_XCPT_16: /* X86_XCPT_MF */
5453 return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
5454
5455 case SVM_EXIT_XCPT_1: /* X86_XCPT_DB */
5456 return hmR0SvmExitXcptDB(pVCpu, pCtx, pSvmTransient);
5457
5458 case SVM_EXIT_XCPT_17: /* X86_XCPT_AC */
5459 return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5460
5461 case SVM_EXIT_XCPT_3: /* X86_XCPT_BP */
5462 return hmR0SvmExitXcptBP(pVCpu, pCtx, pSvmTransient);
5463
5464 case SVM_EXIT_MONITOR:
5465 return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
5466
5467 case SVM_EXIT_MWAIT:
5468 return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
5469
5470 case SVM_EXIT_HLT:
5471 return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
5472
5473 case SVM_EXIT_READ_CR0:
5474 case SVM_EXIT_READ_CR3:
5475 case SVM_EXIT_READ_CR4:
5476 return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5477
5478 case SVM_EXIT_CR0_SEL_WRITE:
5479 case SVM_EXIT_WRITE_CR0:
5480 case SVM_EXIT_WRITE_CR3:
5481 case SVM_EXIT_WRITE_CR4:
5482 case SVM_EXIT_WRITE_CR8:
5483 {
5484 uint8_t const uCr = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : uExitCode - SVM_EXIT_WRITE_CR0;
5485 Log4(("hmR0SvmHandleExit: Write CR%u\n", uCr)); NOREF(uCr);
5486 return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5487 }
5488
5489 case SVM_EXIT_PAUSE:
5490 return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5491
5492 case SVM_EXIT_VMMCALL:
5493 return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5494
5495 case SVM_EXIT_VINTR:
5496 return hmR0SvmExitVIntr(pVCpu, pCtx, pSvmTransient);
5497
5498 case SVM_EXIT_FERR_FREEZE:
5499 return hmR0SvmExitFerrFreeze(pVCpu, pCtx, pSvmTransient);
5500
5501 case SVM_EXIT_INTR:
5502 case SVM_EXIT_NMI:
5503 case SVM_EXIT_XCPT_NMI: /* Shouldn't ever happen, SVM_EXIT_NMI is used instead. */
5504 return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5505
5506 case SVM_EXIT_MSR:
5507 return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
5508
5509 case SVM_EXIT_INVLPG:
5510 return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5511
5512 case SVM_EXIT_WBINVD:
5513 return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5514
5515 case SVM_EXIT_INVD:
5516 return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5517
5518 case SVM_EXIT_RDPMC:
5519 return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5520
5521 default:
5522 {
5523 switch (pSvmTransient->u64ExitCode)
5524 {
5525 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5526 case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5527 case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5528 case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5529 return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5530
5531 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5532 case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5533 case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5534 case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5535 return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5536
5537 case SVM_EXIT_XSETBV:
5538 return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5539
5540 case SVM_EXIT_TASK_SWITCH:
5541 return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5542
5543 case SVM_EXIT_IRET:
5544 return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5545
5546 case SVM_EXIT_SHUTDOWN:
5547 return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5548
5549 case SVM_EXIT_SMI:
5550 case SVM_EXIT_INIT:
5551 {
5552 /*
5553 * We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5554 * If it ever does, we want to know about it so log the exit code and bail.
5555 */
5556 return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5557 }
5558
5559#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5560 case SVM_EXIT_CLGI: return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5561 case SVM_EXIT_STGI: return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5562 case SVM_EXIT_VMLOAD: return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5563 case SVM_EXIT_VMSAVE: return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5564 case SVM_EXIT_INVLPGA: return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5565 case SVM_EXIT_VMRUN: return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5566#else
5567 case SVM_EXIT_CLGI:
5568 case SVM_EXIT_STGI:
5569 case SVM_EXIT_VMLOAD:
5570 case SVM_EXIT_VMSAVE:
5571 case SVM_EXIT_INVLPGA:
5572 case SVM_EXIT_VMRUN:
5573#endif
5574 case SVM_EXIT_RSM:
5575 case SVM_EXIT_SKINIT:
5576 {
5577 hmR0SvmSetPendingXcptUD(pVCpu);
5578 return VINF_SUCCESS;
5579 }
5580
5581#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5582 case SVM_EXIT_XCPT_0: /* #DE */
5583 /* SVM_EXIT_XCPT_1: */ /* #DB - Handled above. */
5584 /* SVM_EXIT_XCPT_2: */ /* #NMI - Handled above. */
5585 /* SVM_EXIT_XCPT_3: */ /* #BP - Handled above. */
5586 case SVM_EXIT_XCPT_4: /* #OF */
5587 case SVM_EXIT_XCPT_5: /* #BR */
5588 /* SVM_EXIT_XCPT_6: */ /* #UD - Handled above. */
5589 case SVM_EXIT_XCPT_7: /* #NM */
5590 case SVM_EXIT_XCPT_8: /* #DF */
5591 case SVM_EXIT_XCPT_9: /* #CO_SEG_OVERRUN */
5592 case SVM_EXIT_XCPT_10: /* #TS */
5593 case SVM_EXIT_XCPT_11: /* #NP */
5594 case SVM_EXIT_XCPT_12: /* #SS */
5595 case SVM_EXIT_XCPT_13: /* #GP */
5596 /* SVM_EXIT_XCPT_14: */ /* #PF - Handled above. */
5597 case SVM_EXIT_XCPT_15: /* Reserved. */
5598 /* SVM_EXIT_XCPT_16: */ /* #MF - Handled above. */
5599 /* SVM_EXIT_XCPT_17: */ /* #AC - Handled above. */
5600 case SVM_EXIT_XCPT_18: /* #MC */
5601 case SVM_EXIT_XCPT_19: /* #XF */
5602 case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5603 case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5604 case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5605 return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5606#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5607
5608 default:
5609 {
5610 AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5611 pVCpu->hm.s.u32HMError = uExitCode;
5612 return VERR_SVM_UNKNOWN_EXIT;
5613 }
5614 }
5615 }
5616 }
5617 /* not reached */
5618}
5619
5620
5621#ifdef DEBUG
5622/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5623# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5624 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5625
5626# define HMSVM_ASSERT_PREEMPT_CPUID() \
5627 do \
5628 { \
5629 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5630 AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5631 } while (0)
5632
5633# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() \
5634 do { \
5635 AssertPtr(pVCpu); \
5636 AssertPtr(pCtx); \
5637 AssertPtr(pSvmTransient); \
5638 Assert(ASMIntAreEnabled()); \
5639 HMSVM_ASSERT_PREEMPT_SAFE(); \
5640 HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5641 Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
5642 HMSVM_ASSERT_PREEMPT_SAFE(); \
5643 if (VMMR0IsLogFlushDisabled(pVCpu)) \
5644 HMSVM_ASSERT_PREEMPT_CPUID(); \
5645 } while (0)
5646#else /* Release builds */
5647# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() do { NOREF(pVCpu); NOREF(pCtx); NOREF(pSvmTransient); } while (0)
5648#endif
5649
5650
5651/**
5652 * Worker for hmR0SvmInterpretInvlpg().
5653 *
5654 * @return VBox status code.
5655 * @param pVCpu The cross context virtual CPU structure.
5656 * @param pCpu Pointer to the disassembler state.
5657 * @param pCtx The guest CPU context.
5658 */
5659static int hmR0SvmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTX pCtx)
5660{
5661 DISQPVPARAMVAL Param1;
5662 RTGCPTR GCPtrPage;
5663
5664 int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), pCpu, &pCpu->Param1, &Param1, DISQPVWHICH_SRC);
5665 if (RT_FAILURE(rc))
5666 return VERR_EM_INTERPRETER;
5667
5668 if ( Param1.type == DISQPV_TYPE_IMMEDIATE
5669 || Param1.type == DISQPV_TYPE_ADDRESS)
5670 {
5671 if (!(Param1.flags & (DISQPV_FLAG_32 | DISQPV_FLAG_64)))
5672 return VERR_EM_INTERPRETER;
5673
5674 GCPtrPage = Param1.val.val64;
5675 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), GCPtrPage);
5676 rc = VBOXSTRICTRC_VAL(rc2);
5677 }
5678 else
5679 {
5680 Log4(("hmR0SvmInterpretInvlPgEx invalid parameter type %#x\n", Param1.type));
5681 rc = VERR_EM_INTERPRETER;
5682 }
5683
5684 return rc;
5685}
5686
5687
5688/**
5689 * Interprets INVLPG.
5690 *
5691 * @returns VBox status code.
5692 * @retval VINF_* Scheduling instructions.
5693 * @retval VERR_EM_INTERPRETER Something we can't cope with.
5694 * @retval VERR_* Fatal errors.
5695 *
5696 * @param pVM The cross context VM structure.
5697 * @param pVCpu The cross context virtual CPU structure.
5698 * @param pCtx The guest CPU context.
5699 *
5700 * @remarks Updates the RIP if the instruction was executed successfully.
5701 */
5702static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5703{
5704 /* Only allow 32 & 64 bit code. */
5705 if (CPUMGetGuestCodeBits(pVCpu) != 16)
5706 {
5707 PDISSTATE pDis = &pVCpu->hm.s.DisState;
5708 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
5709 if ( RT_SUCCESS(rc)
5710 && pDis->pCurInstr->uOpcode == OP_INVLPG)
5711 {
5712 rc = hmR0SvmInterpretInvlPgEx(pVCpu, pDis, pCtx);
5713 if (RT_SUCCESS(rc))
5714 pCtx->rip += pDis->cbInstr;
5715 return rc;
5716 }
5717 else
5718 Log4(("hmR0SvmInterpretInvlpg: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
5719 }
5720 return VERR_EM_INTERPRETER;
5721}
5722
5723
5724#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5725/**
5726 * Gets the IEM exception flags for the specified SVM event.
5727 *
5728 * @returns The IEM exception flags.
5729 * @param pEvent Pointer to the SVM event.
5730 *
5731 * @remarks This function currently only constructs flags required for
5732 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5733 * and CR2 aspects of an exception are not included).
5734 */
5735static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
5736{
5737 uint8_t const uEventType = pEvent->n.u3Type;
5738 uint32_t fIemXcptFlags;
5739 switch (uEventType)
5740 {
5741 case SVM_EVENT_EXCEPTION:
5742 /*
5743 * Only INT3 and INTO instructions can raise #BP and #OF exceptions.
5744 * See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
5745 */
5746 if (pEvent->n.u8Vector == X86_XCPT_BP)
5747 {
5748 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_BP_INSTR;
5749 break;
5750 }
5751 if (pEvent->n.u8Vector == X86_XCPT_OF)
5752 {
5753 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_OF_INSTR;
5754 break;
5755 }
5756 /** @todo How do we distinguish ICEBP \#DB from the regular one? */
5757 RT_FALL_THRU();
5758 case SVM_EVENT_NMI:
5759 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5760 break;
5761
5762 case SVM_EVENT_EXTERNAL_IRQ:
5763 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5764 break;
5765
5766 case SVM_EVENT_SOFTWARE_INT:
5767 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5768 break;
5769
5770 default:
5771 fIemXcptFlags = 0;
5772 AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
5773 break;
5774 }
5775 return fIemXcptFlags;
5776}
5777
5778#else
5779/**
5780 * Determines if an exception is a contributory exception.
5781 *
5782 * Contributory exceptions are ones which can cause double-faults unless the
5783 * original exception was a benign exception. Page-fault is intentionally not
5784 * included here as it's a conditional contributory exception.
5785 *
5786 * @returns @c true if the exception is contributory, @c false otherwise.
5787 * @param uVector The exception vector.
5788 */
5789DECLINLINE(bool) hmR0SvmIsContributoryXcpt(const uint32_t uVector)
5790{
5791 switch (uVector)
5792 {
5793 case X86_XCPT_GP:
5794 case X86_XCPT_SS:
5795 case X86_XCPT_NP:
5796 case X86_XCPT_TS:
5797 case X86_XCPT_DE:
5798 return true;
5799 default:
5800 break;
5801 }
5802 return false;
5803}
5804#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5805
5806
5807/**
5808 * Handle a condition that occurred while delivering an event through the guest
5809 * IDT.
5810 *
5811 * @returns VBox status code (informational error codes included).
5812 * @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
5813 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5814 * continue execution of the guest which will delivery the \#DF.
5815 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5816 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5817 *
5818 * @param pVCpu The cross context virtual CPU structure.
5819 * @param pCtx Pointer to the guest-CPU context.
5820 * @param pSvmTransient Pointer to the SVM transient structure.
5821 *
5822 * @remarks No-long-jump zone!!!
5823 */
5824static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5825{
5826 int rc = VINF_SUCCESS;
5827 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5828
5829 Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
5830 pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
5831 pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
5832
5833 /* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code". The EXITINTINFO (if valid) contains the prior exception (IDT vector)
5834 * that was trying to be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector). */
5835 if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
5836 {
5837#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5838 IEMXCPTRAISE enmRaise;
5839 IEMXCPTRAISEINFO fRaiseInfo;
5840 bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
5841 uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5842 if (fExitIsHwXcpt)
5843 {
5844 uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
5845 uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
5846 uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5847 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5848 }
5849 else
5850 {
5851 /*
5852 * If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF) then we
5853 * end up here.
5854 *
5855 * If the event was:
5856 * - a software interrupt, we can re-execute the instruction which will regenerate
5857 * the event.
5858 * - an NMI, we need to clear NMI blocking and re-inject the NMI.
5859 * - a hardware exception or external interrupt, we re-inject it.
5860 */
5861 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5862 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
5863 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5864 else
5865 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5866 }
5867
5868 switch (enmRaise)
5869 {
5870 case IEMXCPTRAISE_CURRENT_XCPT:
5871 case IEMXCPTRAISE_PREV_EVENT:
5872 {
5873 /* For software interrupts, we shall re-execute the instruction. */
5874 if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
5875 {
5876 RTGCUINTPTR GCPtrFaultAddress = 0;
5877
5878 /* If we are re-injecting an NMI, clear NMI blocking. */
5879 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5880 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5881
5882 /* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
5883 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5884 {
5885 pSvmTransient->fVectoringPF = true;
5886 Log4(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n", pCtx->cr2));
5887 }
5888 else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
5889 && uIdtVector == X86_XCPT_PF)
5890 {
5891 /*
5892 * If the previous exception was a #PF, we need to recover the CR2 value.
5893 * This can't happen with shadow paging.
5894 */
5895 GCPtrFaultAddress = pCtx->cr2;
5896 }
5897
5898 /*
5899 * Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
5900 * exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
5901 */
5902 Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5903 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5904 hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
5905
5906 Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
5907 pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
5908 pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
5909 }
5910 break;
5911 }
5912
5913 case IEMXCPTRAISE_REEXEC_INSTR:
5914 {
5915 Assert(rc == VINF_SUCCESS);
5916 break;
5917 }
5918
5919 case IEMXCPTRAISE_DOUBLE_FAULT:
5920 {
5921 /*
5922 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5923 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5924 */
5925 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5926 {
5927 Log4(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pCtx->cr2));
5928 pSvmTransient->fVectoringDoublePF = true;
5929 Assert(rc == VINF_SUCCESS);
5930 }
5931 else
5932 {
5933 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5934 hmR0SvmSetPendingXcptDF(pVCpu);
5935 rc = VINF_HM_DOUBLE_FAULT;
5936 }
5937 break;
5938 }
5939
5940 case IEMXCPTRAISE_TRIPLE_FAULT:
5941 {
5942 rc = VINF_EM_RESET;
5943 break;
5944 }
5945
5946 case IEMXCPTRAISE_CPU_HANG:
5947 {
5948 rc = VERR_EM_GUEST_CPU_HANG;
5949 break;
5950 }
5951
5952 default:
5953 {
5954 AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
5955 rc = VERR_SVM_IPE_2;
5956 break;
5957 }
5958 }
5959#else
5960 uint8_t uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5961
5962 typedef enum
5963 {
5964 SVMREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5965 SVMREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5966 SVMREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5967 SVMREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5968 SVMREFLECTXCPT_NONE /* Nothing to reflect. */
5969 } SVMREFLECTXCPT;
5970
5971 SVMREFLECTXCPT enmReflect = SVMREFLECTXCPT_NONE;
5972 bool fReflectingNmi = false;
5973 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION)
5974 {
5975 if (pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31)
5976 {
5977 uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0);
5978
5979#ifdef VBOX_STRICT
5980 if ( hmR0SvmIsContributoryXcpt(uIdtVector)
5981 && uExitVector == X86_XCPT_PF)
5982 {
5983 Log4(("IDT: Contributory #PF idCpu=%u uCR2=%#RX64\n", pVCpu->idCpu, pCtx->cr2));
5984 }
5985#endif
5986
5987 if ( uIdtVector == X86_XCPT_BP
5988 || uIdtVector == X86_XCPT_OF)
5989 {
5990 /* Ignore INT3/INTO, just re-execute. See @bugref{8357}. */
5991 }
5992 else if ( uExitVector == X86_XCPT_PF
5993 && uIdtVector == X86_XCPT_PF)
5994 {
5995 pSvmTransient->fVectoringDoublePF = true;
5996 Log4(("IDT: Vectoring double #PF uCR2=%#RX64\n", pCtx->cr2));
5997 }
5998 else if ( uExitVector == X86_XCPT_AC
5999 && uIdtVector == X86_XCPT_AC)
6000 {
6001 enmReflect = SVMREFLECTXCPT_HANG;
6002 Log4(("IDT: Nested #AC - Bad guest\n"));
6003 }
6004 else if ( (pVmcb->ctrl.u32InterceptXcpt & HMSVM_CONTRIBUTORY_XCPT_MASK)
6005 && hmR0SvmIsContributoryXcpt(uExitVector)
6006 && ( hmR0SvmIsContributoryXcpt(uIdtVector)
6007 || uIdtVector == X86_XCPT_PF))
6008 {
6009 enmReflect = SVMREFLECTXCPT_DF;
6010 Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6011 uIdtVector, uExitVector));
6012 }
6013 else if (uIdtVector == X86_XCPT_DF)
6014 {
6015 enmReflect = SVMREFLECTXCPT_TF;
6016 Log4(("IDT: Pending vectoring triple-fault %#RX64 uIdtVector=%#x uExitVector=%#x\n",
6017 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6018 }
6019 else
6020 enmReflect = SVMREFLECTXCPT_XCPT;
6021 }
6022 else
6023 {
6024 /*
6025 * If event delivery caused an #VMEXIT that is not an exception (e.g. #NPF) then reflect the original
6026 * exception to the guest after handling the #VMEXIT.
6027 */
6028 enmReflect = SVMREFLECTXCPT_XCPT;
6029 }
6030 }
6031 else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXTERNAL_IRQ
6032 || pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6033 {
6034 enmReflect = SVMREFLECTXCPT_XCPT;
6035 fReflectingNmi = RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI);
6036
6037 if (pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31)
6038 {
6039 uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0);
6040 if (uExitVector == X86_XCPT_PF)
6041 {
6042 pSvmTransient->fVectoringPF = true;
6043 Log4(("IDT: Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pCtx->cr2));
6044 }
6045 }
6046 }
6047 /* else: Ignore software interrupts (INT n) as they reoccur when restarting the instruction. */
6048
6049 switch (enmReflect)
6050 {
6051 case SVMREFLECTXCPT_XCPT:
6052 {
6053 /* If we are re-injecting the NMI, clear NMI blocking. */
6054 if (fReflectingNmi)
6055 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6056
6057 Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6058 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6059 hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, 0 /* GCPtrFaultAddress */);
6060
6061 /* If uExitVector is #PF, CR2 value will be updated from the VMCB if it's a guest #PF. See hmR0SvmExitXcptPF(). */
6062 Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32\n", pVmcb->ctrl.ExitIntInfo.u,
6063 !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid, pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
6064 break;
6065 }
6066
6067 case SVMREFLECTXCPT_DF:
6068 {
6069 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6070 hmR0SvmSetPendingXcptDF(pVCpu);
6071 rc = VINF_HM_DOUBLE_FAULT;
6072 break;
6073 }
6074
6075 case SVMREFLECTXCPT_TF:
6076 {
6077 rc = VINF_EM_RESET;
6078 break;
6079 }
6080
6081 case SVMREFLECTXCPT_HANG:
6082 {
6083 rc = VERR_EM_GUEST_CPU_HANG;
6084 break;
6085 }
6086
6087 default:
6088 Assert(rc == VINF_SUCCESS);
6089 break;
6090 }
6091#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
6092 }
6093 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET || rc == VERR_EM_GUEST_CPU_HANG);
6094 NOREF(pCtx);
6095 return rc;
6096}
6097
6098
6099/**
6100 * Advances the guest RIP making use of the CPU's NRIP_SAVE feature if
6101 * supported, otherwise advances the RIP by the number of bytes specified in
6102 * @a cb.
6103 *
6104 * @param pVCpu The cross context virtual CPU structure.
6105 * @param pCtx Pointer to the guest-CPU context.
6106 * @param cb RIP increment value in bytes.
6107 *
6108 * @remarks Use this function only from \#VMEXIT's where the NRIP value is valid
6109 * when NRIP_SAVE is supported by the CPU, otherwise use
6110 * hmR0SvmAdvanceRipDumb!
6111 */
6112DECLINLINE(void) hmR0SvmAdvanceRipHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
6113{
6114 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6115 if (fSupportsNextRipSave)
6116 {
6117 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6118 Assert(pVmcb);
6119 Assert(pVmcb->ctrl.u64NextRIP);
6120 AssertRelease(pVmcb->ctrl.u64NextRIP - pCtx->rip == cb); /* temporary, remove later */
6121 pCtx->rip = pVmcb->ctrl.u64NextRIP;
6122 }
6123 else
6124 pCtx->rip += cb;
6125
6126 HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
6127}
6128
6129
6130#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
6131/**
6132 * Gets the length of the current instruction if the CPU supports the NRIP_SAVE
6133 * feature. Otherwise, returns the value in @a cbLikely.
6134 *
6135 * @param pVCpu The cross context virtual CPU structure.
6136 * @param pCtx Pointer to the guest-CPU context.
6137 * @param cbLikely The likely instruction length.
6138 */
6139DECLINLINE(uint8_t) hmR0SvmGetInstrLengthHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint8_t cbLikely)
6140{
6141 Assert(cbLikely <= 15); /* See Intel spec. 2.3.11 "AVX Instruction Length" */
6142 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6143 if (fSupportsNextRipSave)
6144 {
6145 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6146 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6147 Assert(cbInstr == cbLikely);
6148 return cbInstr;
6149 }
6150 return cbLikely;
6151}
6152#endif
6153
6154
6155/**
6156 * Advances the guest RIP by the number of bytes specified in @a cb. This does
6157 * not make use of any hardware features to determine the instruction length.
6158 *
6159 * @param pVCpu The cross context virtual CPU structure.
6160 * @param pCtx Pointer to the guest-CPU context.
6161 * @param cb RIP increment value in bytes.
6162 */
6163DECLINLINE(void) hmR0SvmAdvanceRipDumb(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
6164{
6165 pCtx->rip += cb;
6166 HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
6167}
6168#undef HMSVM_UPDATE_INTR_SHADOW
6169
6170
6171/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6172/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6173/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6174
6175/** @name \#VMEXIT handlers.
6176 * @{
6177 */
6178
6179/**
6180 * \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6181 * signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6182 */
6183HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6184{
6185 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6186
6187 if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6188 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6189 else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6191
6192 /*
6193 * AMD-V has no preemption timer and the generic periodic preemption timer has no way to signal -before- the timer
6194 * fires if the current interrupt is our own timer or a some other host interrupt. We also cannot examine what
6195 * interrupt it is until the host actually take the interrupt.
6196 *
6197 * Going back to executing guest code here unconditionally causes random scheduling problems (observed on an
6198 * AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6199 */
6200 return VINF_EM_RAW_INTERRUPT;
6201}
6202
6203
6204/**
6205 * \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6206 */
6207HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6208{
6209 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6210
6211 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6212 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6213 int rc = VINF_SUCCESS;
6214 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6215 return rc;
6216}
6217
6218
6219/**
6220 * \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6221 */
6222HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6223{
6224 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6225
6226 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6228 int rc = VINF_SUCCESS;
6229 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6230 return rc;
6231}
6232
6233
6234/**
6235 * \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6236 */
6237HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6238{
6239 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6240 PVM pVM = pVCpu->CTX_SUFF(pVM);
6241 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6242 if (RT_LIKELY(rc == VINF_SUCCESS))
6243 {
6244 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6245 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6246 }
6247 else
6248 {
6249 AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6250 rc = VERR_EM_INTERPRETER;
6251 }
6252 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6253 return rc;
6254}
6255
6256
6257/**
6258 * \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6259 */
6260HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6261{
6262 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6263 PVM pVM = pVCpu->CTX_SUFF(pVM);
6264 int rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6265 if (RT_LIKELY(rc == VINF_SUCCESS))
6266 {
6267 pSvmTransient->fUpdateTscOffsetting = true;
6268 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6269 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6270 }
6271 else
6272 {
6273 AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6274 rc = VERR_EM_INTERPRETER;
6275 }
6276 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6277 return rc;
6278}
6279
6280
6281/**
6282 * \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6283 */
6284HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6285{
6286 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6287 int rc = EMInterpretRdtscp(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
6288 if (RT_LIKELY(rc == VINF_SUCCESS))
6289 {
6290 pSvmTransient->fUpdateTscOffsetting = true;
6291 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6292 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6293 }
6294 else
6295 {
6296 AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtscp failed with %Rrc\n", rc));
6297 rc = VERR_EM_INTERPRETER;
6298 }
6299 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
6300 return rc;
6301}
6302
6303
6304/**
6305 * \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6306 */
6307HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6308{
6309 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6310 int rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6311 if (RT_LIKELY(rc == VINF_SUCCESS))
6312 {
6313 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6314 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6315 }
6316 else
6317 {
6318 AssertMsgFailed(("hmR0SvmExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6319 rc = VERR_EM_INTERPRETER;
6320 }
6321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6322 return rc;
6323}
6324
6325
6326/**
6327 * \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
6328 */
6329HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6330{
6331 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6332 PVM pVM = pVCpu->CTX_SUFF(pVM);
6333 Assert(!pVM->hm.s.fNestedPaging);
6334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6335
6336 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6337 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6338 if ( fSupportsDecodeAssists
6339 && fSupportsNextRipSave)
6340 {
6341 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6342 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6343 RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
6344 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
6345 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6346 return VBOXSTRICTRC_VAL(rcStrict);
6347 }
6348
6349 int rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, pCtx); /* Updates RIP if successful. */
6350 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
6351 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6352 return rc;
6353}
6354
6355
6356/**
6357 * \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6358 */
6359HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6360{
6361 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6362
6363 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 1);
6364 int rc = EMShouldContinueAfterHalt(pVCpu, pCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6365 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6366 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
6367 if (rc != VINF_SUCCESS)
6368 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6369 return rc;
6370}
6371
6372
6373/**
6374 * \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6375 */
6376HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6377{
6378 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6379 int rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6380 if (RT_LIKELY(rc == VINF_SUCCESS))
6381 {
6382 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6383 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6384 }
6385 else
6386 {
6387 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6388 rc = VERR_EM_INTERPRETER;
6389 }
6390 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6391 return rc;
6392}
6393
6394
6395/**
6396 * \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6397 */
6398HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6399{
6400 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6401 VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6402 int rc = VBOXSTRICTRC_VAL(rc2);
6403 if ( rc == VINF_EM_HALT
6404 || rc == VINF_SUCCESS)
6405 {
6406 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6407
6408 if ( rc == VINF_EM_HALT
6409 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
6410 {
6411 rc = VINF_SUCCESS;
6412 }
6413 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6414 }
6415 else
6416 {
6417 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
6418 rc = VERR_EM_INTERPRETER;
6419 }
6420 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
6421 ("hmR0SvmExitMwait: EMInterpretMWait failed rc=%Rrc\n", rc));
6422 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
6423 return rc;
6424}
6425
6426
6427/**
6428 * \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6429 * \#VMEXIT.
6430 */
6431HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6432{
6433 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6434 return VINF_EM_RESET;
6435}
6436
6437
6438/**
6439 * \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6440 */
6441HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6442{
6443 RT_NOREF(pCtx);
6444 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6445 AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
6446 pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
6447 RT_NOREF(pVmcb);
6448 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6449 return VERR_SVM_UNEXPECTED_EXIT;
6450}
6451
6452
6453/**
6454 * \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6455 */
6456HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6457{
6458 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6459
6460 Log4(("hmR0SvmExitReadCRx: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6461 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0]);
6462
6463 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6464 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6465 if ( fSupportsDecodeAssists
6466 && fSupportsNextRipSave)
6467 {
6468 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6469 bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6470 if (fMovCRx)
6471 {
6472 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6473 uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6474 uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6475 VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6476 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6477 return VBOXSTRICTRC_VAL(rcStrict);
6478 }
6479 /* else: SMSW instruction, fall back below to IEM for this. */
6480 }
6481
6482 VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6483 int rc = VBOXSTRICTRC_VAL(rc2);
6484 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3,
6485 ("hmR0SvmExitReadCRx: EMInterpretInstruction failed rc=%Rrc\n", rc));
6486 Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6487 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6488 return rc;
6489}
6490
6491
6492/**
6493 * \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6494 */
6495HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6496{
6497 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6498
6499 uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6500 uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
6501 Assert(iCrReg <= 15);
6502
6503 VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6504 bool fDecodedInstr = false;
6505 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6506 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6507 if ( fSupportsDecodeAssists
6508 && fSupportsNextRipSave)
6509 {
6510 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6511 bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6512 if (fMovCRx)
6513 {
6514 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6515 uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6516 Log4(("hmR0SvmExitWriteCRx: Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6517 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6518 fDecodedInstr = true;
6519 }
6520 /* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6521 }
6522
6523 if (!fDecodedInstr)
6524 {
6525 Log4(("hmR0SvmExitWriteCRx: iCrReg=%#x\n", iCrReg));
6526 rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(pCtx), NULL);
6527 if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6528 || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6529 rcStrict = VERR_EM_INTERPRETER;
6530 }
6531
6532 if (rcStrict == VINF_SUCCESS)
6533 {
6534 switch (iCrReg)
6535 {
6536 case 0: /* CR0. */
6537 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6538 break;
6539
6540 case 3: /* CR3. */
6541 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
6542 break;
6543
6544 case 4: /* CR4. */
6545 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
6546 break;
6547
6548 case 8: /* CR8 (TPR). */
6549 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
6550 break;
6551
6552 default:
6553 AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6554 pSvmTransient->u64ExitCode, iCrReg));
6555 break;
6556 }
6557 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6558 }
6559 else
6560 Assert(rcStrict == VERR_EM_INTERPRETER || rcStrict == VINF_PGM_CHANGE_MODE || rcStrict == VINF_PGM_SYNC_CR3);
6561 return VBOXSTRICTRC_TODO(rcStrict);
6562}
6563
6564
6565/**
6566 * \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6567 * \#VMEXIT.
6568 */
6569HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6570{
6571 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6572 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6573 PVM pVM = pVCpu->CTX_SUFF(pVM);
6574
6575 int rc;
6576 if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6577 {
6578 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6579 Log4(("MSR Write: idMsr=%#RX32\n", pCtx->ecx));
6580
6581 /* Handle TPR patching; intercepted LSTAR write. */
6582 if ( pVM->hm.s.fTPRPatchingActive
6583 && pCtx->ecx == MSR_K8_LSTAR)
6584 {
6585 if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6586 {
6587 /* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6588 int rc2 = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6589 AssertRC(rc2);
6590 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
6591 }
6592 rc = VINF_SUCCESS;
6593 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6594 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6595 return rc;
6596 }
6597
6598 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6599 if (fSupportsNextRipSave)
6600 {
6601 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6602 if (RT_LIKELY(rc == VINF_SUCCESS))
6603 {
6604 pCtx->rip = pVmcb->ctrl.u64NextRIP;
6605 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6606 }
6607 else
6608 AssertMsg( rc == VERR_EM_INTERPRETER
6609 || rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: EMInterpretWrmsr failed rc=%Rrc\n", rc));
6610 }
6611 else
6612 {
6613 rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */));
6614 if (RT_LIKELY(rc == VINF_SUCCESS))
6615 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc); /* RIP updated by EMInterpretInstruction(). */
6616 else
6617 AssertMsg( rc == VERR_EM_INTERPRETER
6618 || rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: WrMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6619 }
6620
6621 if (rc == VINF_SUCCESS)
6622 {
6623 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
6624 if ( pCtx->ecx >= MSR_IA32_X2APIC_START
6625 && pCtx->ecx <= MSR_IA32_X2APIC_END)
6626 {
6627 /*
6628 * We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest(). When full APIC register
6629 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCB before
6630 * EMInterpretWrmsr() changes it.
6631 */
6632 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
6633 }
6634 else
6635 {
6636 switch (pCtx->ecx)
6637 {
6638 case MSR_K6_EFER: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR); break;
6639 case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
6640 case MSR_K8_FS_BASE:
6641 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
6642 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
6643 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
6644 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
6645 }
6646 }
6647 }
6648 }
6649 else
6650 {
6651 /* MSR Read access. */
6652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6653 Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ);
6654 Log4(("MSR Read: idMsr=%#RX32\n", pCtx->ecx));
6655
6656 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6657 if (fSupportsNextRipSave)
6658 {
6659 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6660 if (RT_LIKELY(rc == VINF_SUCCESS))
6661 {
6662 pCtx->rip = pVmcb->ctrl.u64NextRIP;
6663 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6664 }
6665 else
6666 AssertMsg( rc == VERR_EM_INTERPRETER
6667 || rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: EMInterpretRdmsr failed rc=%Rrc\n", rc));
6668 }
6669 else
6670 {
6671 rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0));
6672 if (RT_UNLIKELY(rc != VINF_SUCCESS))
6673 {
6674 AssertMsg( rc == VERR_EM_INTERPRETER
6675 || rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: RdMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6676 }
6677 /* RIP updated by EMInterpretInstruction(). */
6678 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6679 }
6680 }
6681
6682 /* RIP has been updated by EMInterpret[Rd|Wr]msr() or EMInterpretInstruction(). */
6683 return rc;
6684}
6685
6686
6687/**
6688 * \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6689 */
6690HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6691{
6692 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6694
6695 /** @todo Stepping with nested-guest. */
6696 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6697 {
6698 /* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6699 if (pSvmTransient->fWasGuestDebugStateActive)
6700 {
6701 AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6702 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6703 return VERR_SVM_UNEXPECTED_EXIT;
6704 }
6705
6706 /*
6707 * Lazy DR0-3 loading.
6708 */
6709 if (!pSvmTransient->fWasHyperDebugStateActive)
6710 {
6711 Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6712 Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6713
6714 /* Don't intercept DRx read and writes. */
6715 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6716 pVmcb->ctrl.u16InterceptRdDRx = 0;
6717 pVmcb->ctrl.u16InterceptWrDRx = 0;
6718 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6719
6720 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6721 VMMRZCallRing3Disable(pVCpu);
6722 HM_DISABLE_PREEMPT();
6723
6724 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6725 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6726 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
6727
6728 HM_RESTORE_PREEMPT();
6729 VMMRZCallRing3Enable(pVCpu);
6730
6731 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6732 return VINF_SUCCESS;
6733 }
6734 }
6735
6736 /*
6737 * Interpret the read/writing of DRx.
6738 */
6739 /** @todo Decode assist. */
6740 VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6741 Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6742 if (RT_LIKELY(rc == VINF_SUCCESS))
6743 {
6744 /* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6745 /** @todo CPUM should set this flag! */
6746 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6747 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6748 }
6749 else
6750 Assert(rc == VERR_EM_INTERPRETER);
6751 return VBOXSTRICTRC_TODO(rc);
6752}
6753
6754
6755/**
6756 * \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6757 */
6758HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6759{
6760 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6761 /* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6762 int rc = hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
6763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6764 STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6765 return rc;
6766}
6767
6768
6769/**
6770 * \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6771 */
6772HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6773{
6774 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6775
6776 /** @todo decode assists... */
6777 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6778 if (rcStrict == VINF_IEM_RAISED_XCPT)
6779 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6780
6781 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6782 Log4(("hmR0SvmExitXsetbv: New XCR0=%#RX64 fLoadSaveGuestXcr0=%d (cr4=%RX64) rcStrict=%Rrc\n",
6783 pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0, pCtx->cr4, VBOXSTRICTRC_VAL(rcStrict)));
6784
6785 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6786 return VBOXSTRICTRC_TODO(rcStrict);
6787}
6788
6789
6790/**
6791 * \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6792 */
6793HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6794{
6795 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6796
6797 /* I/O operation lookup arrays. */
6798 static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
6799 static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
6800 the result (in AL/AX/EAX). */
6801 Log4(("hmR0SvmExitIOInstr: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6802
6803 PVM pVM = pVCpu->CTX_SUFF(pVM);
6804 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6805
6806 /* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
6807 SVMIOIOEXITINFO IoExitInfo;
6808 IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
6809 uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
6810 uint32_t cbValue = s_aIOSize[uIOWidth];
6811 uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
6812
6813 if (RT_UNLIKELY(!cbValue))
6814 {
6815 AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
6816 return VERR_EM_INTERPRETER;
6817 }
6818
6819 VBOXSTRICTRC rcStrict;
6820 bool fUpdateRipAlready = false;
6821 if (IoExitInfo.n.u1Str)
6822 {
6823#ifdef VBOX_WITH_2ND_IEM_STEP
6824 /* INS/OUTS - I/O String instruction. */
6825 /** @todo Huh? why can't we use the segment prefix information given by AMD-V
6826 * in EXITINFO1? Investigate once this thing is up and running. */
6827 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
6828 IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
6829 AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
6830 static IEMMODE const s_aenmAddrMode[8] =
6831 {
6832 (IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
6833 };
6834 IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
6835 if (enmAddrMode != (IEMMODE)-1)
6836 {
6837 uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6838 if (cbInstr <= 15 && cbInstr >= 1)
6839 {
6840 Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
6841 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6842 {
6843 /* Don't know exactly how to detect whether u3Seg is valid, currently
6844 only enabling it for Bulldozer and later with NRIP. OS/2 broke on
6845 2384 Opterons when only checking NRIP. */
6846 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6847 if ( fSupportsNextRipSave
6848 && pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
6849 {
6850 AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS || cbInstr > 1U + IoExitInfo.n.u1Rep,
6851 ("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
6852 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6853 IoExitInfo.n.u3Seg, true /*fIoChecked*/);
6854 }
6855 else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
6856 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6857 X86_SREG_DS, true /*fIoChecked*/);
6858 else
6859 rcStrict = IEMExecOne(pVCpu);
6860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6861 }
6862 else
6863 {
6864 AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /*=0*/, ("%#x\n", IoExitInfo.n.u3Seg));
6865 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
6866 true /*fIoChecked*/);
6867 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6868 }
6869 }
6870 else
6871 {
6872 AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
6873 rcStrict = IEMExecOne(pVCpu);
6874 }
6875 }
6876 else
6877 {
6878 AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
6879 rcStrict = IEMExecOne(pVCpu);
6880 }
6881 fUpdateRipAlready = true;
6882
6883#else
6884 /* INS/OUTS - I/O String instruction. */
6885 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
6886
6887 /** @todo Huh? why can't we use the segment prefix information given by AMD-V
6888 * in EXITINFO1? Investigate once this thing is up and running. */
6889
6890 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
6891 if (rcStrict == VINF_SUCCESS)
6892 {
6893 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6894 {
6895 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6896 (DISCPUMODE)pDis->uAddrMode, cbValue);
6897 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6898 }
6899 else
6900 {
6901 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6902 (DISCPUMODE)pDis->uAddrMode, cbValue);
6903 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6904 }
6905 }
6906 else
6907 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
6908#endif
6909 }
6910 else
6911 {
6912 /* IN/OUT - I/O instruction. */
6913 Assert(!IoExitInfo.n.u1Rep);
6914
6915 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6916 {
6917 rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
6918 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
6919 }
6920 else
6921 {
6922 uint32_t u32Val = 0;
6923 rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
6924 if (IOM_SUCCESS(rcStrict))
6925 {
6926 /* Save result of I/O IN instr. in AL/AX/EAX. */
6927 /** @todo r=bird: 32-bit op size should clear high bits of rax! */
6928 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
6929 }
6930 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6931 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
6932
6933 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
6934 }
6935 }
6936
6937 if (IOM_SUCCESS(rcStrict))
6938 {
6939 /* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
6940 if (!fUpdateRipAlready)
6941 pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
6942
6943 /*
6944 * If any I/O breakpoints are armed, we need to check if one triggered
6945 * and take appropriate action.
6946 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
6947 */
6948 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
6949 * execution engines about whether hyper BPs and such are pending. */
6950 uint32_t const uDr7 = pCtx->dr[7];
6951 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
6952 && X86_DR7_ANY_RW_IO(uDr7)
6953 && (pCtx->cr4 & X86_CR4_DE))
6954 || DBGFBpIsHwIoArmed(pVM)))
6955 {
6956 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6957 VMMRZCallRing3Disable(pVCpu);
6958 HM_DISABLE_PREEMPT();
6959
6960 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
6961 CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /*fDr6*/);
6962
6963 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
6964 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
6965 {
6966 /* Raise #DB. */
6967 pVmcb->guest.u64DR6 = pCtx->dr[6];
6968 pVmcb->guest.u64DR7 = pCtx->dr[7];
6969 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
6970 hmR0SvmSetPendingXcptDB(pVCpu);
6971 }
6972 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
6973 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
6974 else if ( rcStrict2 != VINF_SUCCESS
6975 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
6976 rcStrict = rcStrict2;
6977 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
6978
6979 HM_RESTORE_PREEMPT();
6980 VMMRZCallRing3Enable(pVCpu);
6981 }
6982
6983 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6984 }
6985
6986#ifdef VBOX_STRICT
6987 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6988 Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
6989 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
6990 Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
6991 else
6992 {
6993 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
6994 * statuses, that the VMM device and some others may return. See
6995 * IOM_SUCCESS() for guidance. */
6996 AssertMsg( RT_FAILURE(rcStrict)
6997 || rcStrict == VINF_SUCCESS
6998 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
6999 || rcStrict == VINF_EM_DBG_BREAKPOINT
7000 || rcStrict == VINF_EM_RAW_GUEST_TRAP
7001 || rcStrict == VINF_EM_RAW_TO_R3
7002 || rcStrict == VINF_TRPM_XCPT_DISPATCHED
7003 || rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7004 }
7005#endif
7006 return VBOXSTRICTRC_TODO(rcStrict);
7007}
7008
7009
7010/**
7011 * \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7012 */
7013HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7014{
7015 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7016
7017 PVM pVM = pVCpu->CTX_SUFF(pVM);
7018 Assert(pVM->hm.s.fNestedPaging);
7019
7020 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7021
7022 /* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7023 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7024 RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7025 uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7026 thus intentionally not copied into u32ErrCode. */
7027
7028 Log4(("#NPF at CS:RIP=%04x:%#RX64 faultaddr=%RGp errcode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr, u32ErrCode));
7029
7030 /*
7031 * TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7032 */
7033 if ( pVM->hm.s.fTprPatchingAllowed
7034 && (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7035 && ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7036 || (u32ErrCode & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) /* MMIO page. */
7037 && !CPUMIsGuestInLongModeEx(pCtx)
7038 && !CPUMGetGuestCPL(pVCpu)
7039 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7040 {
7041 RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7042 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7043
7044 if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7045 {
7046 /* Only attempt to patch the instruction once. */
7047 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7048 if (!pPatch)
7049 return VINF_EM_HM_PATCH_TPR_INSTR;
7050 }
7051 }
7052
7053 /*
7054 * Determine the nested paging mode.
7055 */
7056 PGMMODE enmNestedPagingMode;
7057#if HC_ARCH_BITS == 32
7058 if (CPUMIsGuestInLongModeEx(pCtx))
7059 enmNestedPagingMode = PGMMODE_AMD64_NX;
7060 else
7061#endif
7062 enmNestedPagingMode = PGMGetHostMode(pVM);
7063
7064 /*
7065 * MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7066 */
7067 int rc;
7068 Assert((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7069 if ((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
7070 {
7071 /* If event delivery causes an MMIO #NPF, go back to instruction emulation as
7072 otherwise injecting the original pending event would most likely cause the same MMIO #NPF. */
7073 if (pVCpu->hm.s.Event.fPending)
7074 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7075
7076 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7077 u32ErrCode);
7078 rc = VBOXSTRICTRC_VAL(rc2);
7079
7080 /*
7081 * If we succeed, resume guest execution.
7082 * If we fail in interpreting the instruction because we couldn't get the guest physical address
7083 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
7084 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
7085 * weird case. See @bugref{6043}.
7086 */
7087 if ( rc == VINF_SUCCESS
7088 || rc == VERR_PAGE_TABLE_NOT_PRESENT
7089 || rc == VERR_PAGE_NOT_PRESENT)
7090 {
7091 /* Successfully handled MMIO operation. */
7092 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
7093 rc = VINF_SUCCESS;
7094 }
7095 return rc;
7096 }
7097
7098 TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
7099 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
7100 TRPMResetTrap(pVCpu);
7101
7102 Log4(("#NPF: PGMR0Trap0eHandlerNestedPaging returned %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
7103
7104 /*
7105 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
7106 */
7107 if ( rc == VINF_SUCCESS
7108 || rc == VERR_PAGE_TABLE_NOT_PRESENT
7109 || rc == VERR_PAGE_NOT_PRESENT)
7110 {
7111 /* We've successfully synced our shadow page tables. */
7112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7113 rc = VINF_SUCCESS;
7114 }
7115
7116 return rc;
7117}
7118
7119
7120/**
7121 * \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
7122 * \#VMEXIT.
7123 */
7124HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7125{
7126 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7127 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7128
7129 /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7130 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7131 hmR0SvmClearIntWindowExiting(pVCpu, pVmcb, pCtx);
7132
7133 /* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7135 return VINF_SUCCESS;
7136}
7137
7138
7139/**
7140 * \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
7141 * \#VMEXIT.
7142 */
7143HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7144{
7145 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7146
7147 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7148
7149#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
7150 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
7151#endif
7152
7153 /* Check if this task-switch occurred while delivering an event through the guest IDT. */
7154 if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
7155 {
7156 /*
7157 * AMD-V provides us with the exception which caused the TS; we collect
7158 * the information in the call to hmR0SvmCheckExitDueToEventDelivery.
7159 */
7160 Log4(("hmR0SvmExitTaskSwitch: TS occurred during event delivery.\n"));
7161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7162 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7163 }
7164
7165 /** @todo Emulate task switch someday, currently just going back to ring-3 for
7166 * emulation. */
7167 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7168 return VERR_EM_INTERPRETER;
7169}
7170
7171/**
7172 * Performs the operations necessary that are part of the vmmcall instruction
7173 * execution in the guest.
7174 *
7175 * @returns Strict VBox status code (i.e. informational status codes too).
7176 * @retval VINF_SUCCESS on successful handling, no \#UD needs to be thrown,
7177 * update RIP and eflags.RF depending on @a pfUpdatedRipAndRF and
7178 * continue guest execution.
7179 * @retval VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
7180 * RIP.
7181 * @retval VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
7182 *
7183 * @param pVCpu The cross context virtual CPU structure.
7184 * @param pCtx Pointer to the guest-CPU context.
7185 * @param pfUpdatedRipAndRF Whether the guest RIP/EIP has been updated as
7186 * part of handling the VMMCALL operation.
7187 *
7188 * @todo r=bird: merge this with hmR0SvmExitVmmCall and fix todos.
7189 */
7190static VBOXSTRICTRC hmR0SvmVmmcall(PVMCPU pVCpu, PCPUMCTX pCtx, bool *pfUpdatedRipAndRF)
7191{
7192 /*
7193 * TPR patched instruction emulation for 32-bit guests.
7194 */
7195 PVM pVM = pVCpu->CTX_SUFF(pVM);
7196 if (pVM->hm.s.fTprPatchingAllowed)
7197 {
7198 int rc = hmSvmEmulateMovTpr(pVCpu, pCtx);
7199 if (RT_SUCCESS(rc))
7200 {
7201 *pfUpdatedRipAndRF = true;
7202 return VINF_SUCCESS;
7203 }
7204
7205 if (rc != VERR_NOT_FOUND)
7206 {
7207 Log(("hmSvmExitVmmCall: hmSvmEmulateMovTpr returns %Rrc\n", rc));
7208 *pfUpdatedRipAndRF = false;
7209 return rc;
7210 }
7211 }
7212
7213 /*
7214 * Paravirtualized hypercalls.
7215 */
7216 *pfUpdatedRipAndRF = false; /** @todo r=bird: This is misleading/wrong, see GIMHypercall returncode docs. */
7217 if (EMAreHypercallInstructionsEnabled(pVCpu))
7218 return GIMHypercall(pVCpu, pCtx);
7219
7220 return VERR_NOT_AVAILABLE;
7221}
7222
7223
7224
7225/**
7226 * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7227 */
7228HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7229{
7230 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7231 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
7232
7233 bool fRipUpdated;
7234 VBOXSTRICTRC rcStrict = hmR0SvmVmmcall(pVCpu, pCtx, &fRipUpdated);
7235 if (RT_SUCCESS(rcStrict))
7236 {
7237 /* Only update the RIP if we're continuing guest execution and not
7238 in the case of say VINF_GIM_R3_HYPERCALL. */
7239 if ( rcStrict == VINF_SUCCESS
7240 && !fRipUpdated)
7241 {
7242 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3 /* cbInstr */);
7243 }
7244
7245 /* If the hypercall or TPR patching changes anything other than guest's general-purpose registers,
7246 we would need to reload the guest changed bits here before VM-entry. */
7247 return VBOXSTRICTRC_VAL(rcStrict);
7248 }
7249
7250 hmR0SvmSetPendingXcptUD(pVCpu);
7251 return VINF_SUCCESS;
7252}
7253
7254
7255/**
7256 * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7257 */
7258HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7259{
7260 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7261 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7262 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
7263 /** @todo The guest has likely hit a contended spinlock. We might want to
7264 * poke a schedule different guest VCPU. */
7265 return VINF_EM_RAW_INTERRUPT;
7266}
7267
7268
7269/**
7270 * \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
7271 * \#VMEXIT.
7272 */
7273HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7274{
7275 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7276 Assert(!(pCtx->cr0 & X86_CR0_NE));
7277
7278 Log4(("hmR0SvmExitFerrFreeze: Raising IRQ 13 in response to #FERR\n"));
7279 return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
7280}
7281
7282
7283/**
7284 * \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7285 */
7286HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7287{
7288 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7289
7290 /* Clear NMI blocking. */
7291 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
7292 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7293
7294 /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7295 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7296 hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_IRET);
7297
7298 /* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7299 return VINF_SUCCESS;
7300}
7301
7302
7303/**
7304 * \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
7305 * Conditional \#VMEXIT.
7306 */
7307HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7308{
7309 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7310
7311 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7312
7313 /* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7314 PVM pVM = pVCpu->CTX_SUFF(pVM);
7315 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7316 uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
7317 uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7318
7319#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(HMSVM_ALWAYS_TRAP_PF)
7320 if (pVM->hm.s.fNestedPaging)
7321 {
7322 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7323 if ( !pSvmTransient->fVectoringDoublePF
7324 || CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7325 {
7326 /* A genuine guest #PF, reflect it to the guest. */
7327 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, uErrCode, uFaultAddress);
7328 Log4(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7329 uFaultAddress, uErrCode));
7330 }
7331 else
7332 {
7333 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7334 hmR0SvmSetPendingXcptDF(pVCpu);
7335 Log4(("Pending #DF due to vectoring #PF. NP\n"));
7336 }
7337 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7338 return VINF_SUCCESS;
7339 }
7340#endif
7341
7342 Assert(!pVM->hm.s.fNestedPaging);
7343
7344 /*
7345 * TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
7346 */
7347 if ( pVM->hm.s.fTprPatchingAllowed
7348 && (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7349 && !(uErrCode & X86_TRAP_PF_P) /* Not present. */
7350 && !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7351 && !CPUMIsGuestInLongModeEx(pCtx)
7352 && !CPUMGetGuestCPL(pVCpu)
7353 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7354 {
7355 RTGCPHYS GCPhysApicBase;
7356 GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7357 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7358
7359 /* Check if the page at the fault-address is the APIC base. */
7360 RTGCPHYS GCPhysPage;
7361 int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7362 if ( rc2 == VINF_SUCCESS
7363 && GCPhysPage == GCPhysApicBase)
7364 {
7365 /* Only attempt to patch the instruction once. */
7366 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7367 if (!pPatch)
7368 return VINF_EM_HM_PATCH_TPR_INSTR;
7369 }
7370 }
7371
7372 Log4(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7373 pCtx->rip, uErrCode, pCtx->cr3));
7374
7375 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7376 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7377 if (pSvmTransient->fVectoringPF)
7378 {
7379 Assert(pVCpu->hm.s.Event.fPending);
7380 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7381 }
7382
7383 TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
7384 int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7385
7386 Log4(("#PF: rc=%Rrc\n", rc));
7387
7388 if (rc == VINF_SUCCESS)
7389 {
7390 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7391 TRPMResetTrap(pVCpu);
7392 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7393 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7394 return rc;
7395 }
7396
7397 if (rc == VINF_EM_RAW_GUEST_TRAP)
7398 {
7399 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7400
7401 /*
7402 * If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
7403 * we should simply forward the #PF to the guest and is up to the nested-hypervisor to
7404 * determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
7405 */
7406 if ( !pSvmTransient->fVectoringDoublePF
7407 || CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7408 {
7409 /* It's a guest (or nested-guest) page fault and needs to be reflected. */
7410 uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7411 TRPMResetTrap(pVCpu);
7412
7413#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7414 /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
7415 if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7416 && HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
7417 return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress));
7418#endif
7419
7420 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, uErrCode, uFaultAddress);
7421 }
7422 else
7423 {
7424 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7425 TRPMResetTrap(pVCpu);
7426 hmR0SvmSetPendingXcptDF(pVCpu);
7427 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7428 }
7429
7430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7431 return VINF_SUCCESS;
7432 }
7433
7434 TRPMResetTrap(pVCpu);
7435 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7436 return rc;
7437}
7438
7439
7440/**
7441 * \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
7442 * Conditional \#VMEXIT.
7443 */
7444HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7445{
7446 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7447 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7448
7449 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
7450 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7451 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7452
7453 int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7454 if (pVCpu->hm.s.fGIMTrapXcptUD)
7455 {
7456 uint8_t cbInstr = 0;
7457 VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, pCtx, NULL /* pDis */, &cbInstr);
7458 if (rcStrict == VINF_SUCCESS)
7459 {
7460 /* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7461 hmR0SvmAdvanceRipDumb(pVCpu, pCtx, cbInstr);
7462 rc = VINF_SUCCESS;
7463 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7464 }
7465 else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7466 rc = VINF_SUCCESS;
7467 else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7468 rc = VINF_GIM_R3_HYPERCALL;
7469 else
7470 Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7471 }
7472
7473 /* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7474 if (RT_FAILURE(rc))
7475 {
7476 hmR0SvmSetPendingXcptUD(pVCpu);
7477 rc = VINF_SUCCESS;
7478 }
7479
7480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7481 return rc;
7482}
7483
7484
7485/**
7486 * \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
7487 * Conditional \#VMEXIT.
7488 */
7489HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7490{
7491 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7492
7493 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
7494 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7495 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7496
7497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7498
7499 if (!(pCtx->cr0 & X86_CR0_NE))
7500 {
7501 PVM pVM = pVCpu->CTX_SUFF(pVM);
7502 PDISSTATE pDis = &pVCpu->hm.s.DisState;
7503 unsigned cbOp;
7504 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
7505 if (RT_SUCCESS(rc))
7506 {
7507 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7508 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
7509 if (RT_SUCCESS(rc))
7510 pCtx->rip += cbOp;
7511 }
7512 else
7513 Log4(("hmR0SvmExitXcptMF: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7514 return rc;
7515 }
7516
7517 hmR0SvmSetPendingXcptMF(pVCpu);
7518 return VINF_SUCCESS;
7519}
7520
7521
7522/**
7523 * \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
7524 * \#VMEXIT.
7525 */
7526HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7527{
7528 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7529
7530 /* If this #DB is the result of delivering an event, go back to the interpreter. */
7531 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7532 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7533 {
7534 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7535 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7536 }
7537
7538 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7539
7540 /* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data breakpoint). However, for both cases
7541 DR6 and DR7 are updated to what the exception handler expects. See AMD spec. 15.12.2 "#DB (Debug)". */
7542 PVM pVM = pVCpu->CTX_SUFF(pVM);
7543 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7544 int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7545 if (rc == VINF_EM_RAW_GUEST_TRAP)
7546 {
7547 Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7548 if (CPUMIsHyperDebugStateActive(pVCpu))
7549 CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) | pVmcb->guest.u64DR6);
7550
7551 /* Reflect the exception back to the guest. */
7552 hmR0SvmSetPendingXcptDB(pVCpu);
7553 rc = VINF_SUCCESS;
7554 }
7555
7556 /*
7557 * Update DR6.
7558 */
7559 if (CPUMIsHyperDebugStateActive(pVCpu))
7560 {
7561 Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7562 pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7563 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7564 }
7565 else
7566 {
7567 AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7568 Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7569 }
7570
7571 return rc;
7572}
7573
7574
7575/**
7576 * \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
7577 * Conditional \#VMEXIT.
7578 */
7579HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7580{
7581 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7582
7583 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7584
7585 SVMEVENT Event;
7586 Event.u = 0;
7587 Event.n.u1Valid = 1;
7588 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7589 Event.n.u8Vector = X86_XCPT_AC;
7590 Event.n.u1ErrorCodeValid = 1;
7591 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7592 return VINF_SUCCESS;
7593}
7594
7595
7596/**
7597 * \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
7598 * Conditional \#VMEXIT.
7599 */
7600HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7601{
7602 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7603
7604 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7605
7606 int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7607 if (rc == VINF_EM_RAW_GUEST_TRAP)
7608 {
7609 SVMEVENT Event;
7610 Event.u = 0;
7611 Event.n.u1Valid = 1;
7612 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7613 Event.n.u8Vector = X86_XCPT_BP;
7614 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7615 }
7616
7617 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
7618 return rc;
7619}
7620
7621
7622#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(VBOX_WITH_NESTED_HWVIRT_SVM)
7623/**
7624 * \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7625 */
7626HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7627{
7628 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7629
7630 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7631
7632 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7633 uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
7634 uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7635 Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7636 Assert(uVector <= X86_XCPT_LAST);
7637 Log4(("hmR0SvmExitXcptGeneric: uVector=%#x uErrCode=%u\n", uVector, uErrCode));
7638
7639 SVMEVENT Event;
7640 Event.u = 0;
7641 Event.n.u1Valid = 1;
7642 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7643 Event.n.u8Vector = uVector;
7644 switch (uVector)
7645 {
7646 /* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
7647 case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
7648 case X86_XCPT_DF:
7649 case X86_XCPT_TS:
7650 case X86_XCPT_NP:
7651 case X86_XCPT_SS:
7652 case X86_XCPT_GP:
7653 case X86_XCPT_AC:
7654 {
7655 Event.n.u1ErrorCodeValid = 1;
7656 Event.n.u32ErrorCode = uErrCode;
7657 break;
7658 }
7659 }
7660
7661 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7662 return VINF_SUCCESS;
7663}
7664#endif
7665
7666#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
7667/**
7668 * \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7669 */
7670HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7671{
7672 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7673
7674#ifdef VBOX_STRICT
7675 PCSVMVMCB pVmcbTmp = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7676 Assert(pVmcbTmp);
7677 Assert(!pVmcbTmp->ctrl.IntCtrl.n.u1VGifEnable);
7678 RT_NOREF(pVmcbTmp);
7679#endif
7680
7681 /** @todo Stat. */
7682 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClgi); */
7683 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7684 VBOXSTRICTRC rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7685 return VBOXSTRICTRC_VAL(rcStrict);
7686}
7687
7688
7689/**
7690 * \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7691 */
7692HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7693{
7694 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7695
7696 /*
7697 * When VGIF is not used we always intercept STGI instructions. When VGIF is used,
7698 * we only intercept STGI when events are pending for GIF to become 1.
7699 */
7700 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7701 if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
7702 hmR0SvmClearCtrlIntercept(pVCpu, pCtx, pVmcb, SVM_CTRL_INTERCEPT_STGI);
7703
7704 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7705 VBOXSTRICTRC rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7706 return VBOXSTRICTRC_VAL(rcStrict);
7707}
7708
7709
7710/**
7711 * \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7712 */
7713HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7714{
7715 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7716
7717#ifdef VBOX_STRICT
7718 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7719 Assert(pVmcb);
7720 Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7721 RT_NOREF(pVmcb);
7722#endif
7723
7724 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7725 VBOXSTRICTRC rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7726 if (rcStrict == VINF_SUCCESS)
7727 {
7728 /* We skip flagging changes made to LSTAR, STAR, SFMASK and other MSRs as they are always re-loaded. */
7729 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7730 | HM_CHANGED_GUEST_TR
7731 | HM_CHANGED_GUEST_LDTR);
7732 }
7733 return VBOXSTRICTRC_VAL(rcStrict);
7734}
7735
7736
7737/**
7738 * \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7739 */
7740HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7741{
7742 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7743
7744#ifdef VBOX_STRICT
7745 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7746 Assert(pVmcb);
7747 Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7748 RT_NOREF(pVmcb);
7749#endif
7750
7751 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7752 VBOXSTRICTRC rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7753 return VBOXSTRICTRC_VAL(rcStrict);
7754}
7755
7756
7757/**
7758 * \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
7759 */
7760HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7761{
7762 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7763 /** @todo Stat. */
7764 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpga); */
7765 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7766 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
7767 return VBOXSTRICTRC_VAL(rcStrict);
7768}
7769
7770
7771/**
7772 * \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
7773 */
7774HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7775{
7776 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7777
7778 VBOXSTRICTRC rcStrict;
7779 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7780 rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
7781 Log4(("IEMExecDecodedVmrun: returned %d\n", VBOXSTRICTRC_VAL(rcStrict)));
7782 if (rcStrict == VINF_SUCCESS)
7783 {
7784 rcStrict = VINF_SVM_VMRUN;
7785 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7786 }
7787 return VBOXSTRICTRC_VAL(rcStrict);
7788}
7789
7790
7791/**
7792 * Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
7793 * Unconditional \#VMEXIT.
7794 */
7795HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7796{
7797 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7798
7799 /* If this #DB is the result of delivering an event, go back to the interpreter. */
7800 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7801 if (pVCpu->hm.s.Event.fPending)
7802 {
7803 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7804 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7805 }
7806
7807 hmR0SvmSetPendingXcptDB(pVCpu);
7808 return VINF_SUCCESS;
7809}
7810
7811
7812/**
7813 * Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
7814 * Conditional \#VMEXIT.
7815 */
7816HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7817{
7818 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7819
7820 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7821
7822 SVMEVENT Event;
7823 Event.u = 0;
7824 Event.n.u1Valid = 1;
7825 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7826 Event.n.u8Vector = X86_XCPT_BP;
7827 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7828 return VINF_SUCCESS;
7829}
7830
7831#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
7832
7833
7834/** @} */
7835
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette