VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 13514

Last change on this file since 13514 was 13514, checked in by vboxsync, 16 years ago

Enabled power notification callbacks to disable VT-x/AMD-V before suspend and to re-enable it after resume.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 88.3 KB
Line 
1/* $Id: HWSVMR0.cpp 13514 2008-10-23 07:56:55Z vboxsync $ */
2/** @file
3 * HWACCM SVM - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_HWACCM
27#include <VBox/hwaccm.h>
28#include "HWACCMInternal.h"
29#include <VBox/vm.h>
30#include <VBox/x86.h>
31#include <VBox/hwacc_svm.h>
32#include <VBox/pgm.h>
33#include <VBox/pdm.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <VBox/selm.h>
37#include <VBox/iom.h>
38#include <VBox/dis.h>
39#include <VBox/dbgf.h>
40#include <VBox/disopcode.h>
41#include <iprt/param.h>
42#include <iprt/assert.h>
43#include <iprt/asm.h>
44#include <iprt/cpuset.h>
45#include <iprt/mp.h>
46#include "HWSVMR0.h"
47
48/*******************************************************************************
49* Internal Functions *
50*******************************************************************************/
51static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID);
52
53/*******************************************************************************
54* Global Variables *
55*******************************************************************************/
56/* IO operation lookup arrays. */
57static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
58
59/**
60 * Sets up and activates AMD-V on the current CPU
61 *
62 * @returns VBox status code.
63 * @param pCpu CPU info struct
64 * @param pVM The VM to operate on. (can be NULL after a resume!!)
65 * @param pvPageCpu Pointer to the global cpu page
66 * @param pPageCpuPhys Physical address of the global cpu page
67 */
68VMMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
69{
70 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
71 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
72
73 /* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
74
75#ifdef LOG_ENABLED
76 SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
77#endif
78
79 /* Turn on AMD-V in the EFER MSR. */
80 uint64_t val = ASMRdMsr(MSR_K6_EFER);
81 if (!(val & MSR_K6_EFER_SVME))
82 ASMWrMsr(MSR_K6_EFER, val | MSR_K6_EFER_SVME);
83
84 /* Write the physical page address where the CPU will store the host state while executing the VM. */
85 ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
86
87 return VINF_SUCCESS;
88}
89
90/**
91 * Deactivates AMD-V on the current CPU
92 *
93 * @returns VBox status code.
94 * @param pCpu CPU info struct
95 * @param pvPageCpu Pointer to the global cpu page
96 * @param pPageCpuPhys Physical address of the global cpu page
97 */
98VMMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
99{
100 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
101 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
102
103#ifdef LOG_ENABLED
104 SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
105#endif
106
107 /* Turn off AMD-V in the EFER MSR. */
108 uint64_t val = ASMRdMsr(MSR_K6_EFER);
109 ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
110
111 /* Invalidate host state physical address. */
112 ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
113
114 return VINF_SUCCESS;
115}
116
117/**
118 * Does Ring-0 per VM AMD-V init.
119 *
120 * @returns VBox status code.
121 * @param pVM The VM to operate on.
122 */
123VMMR0DECL(int) SVMR0InitVM(PVM pVM)
124{
125 int rc;
126
127 pVM->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
128 pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
129 pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
130 pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
131
132
133 /* Allocate one page for the VM control block (VMCB). */
134 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
135 if (RT_FAILURE(rc))
136 return rc;
137
138 pVM->hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCB);
139 pVM->hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCB, 0);
140 ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCB);
141
142 /* Allocate one page for the host context */
143 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
144 if (RT_FAILURE(rc))
145 return rc;
146
147 pVM->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCBHost);
148 pVM->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCBHost, 0);
149 ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCBHost);
150
151 /* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
152 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
153 if (RT_FAILURE(rc))
154 return rc;
155
156 pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
157 pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
158 /* Set all bits to intercept all IO accesses. */
159 ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
160
161 /* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
162 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
163 if (RT_FAILURE(rc))
164 return rc;
165
166 pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
167 pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
168 /* Set all bits to intercept all MSR accesses. */
169 ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
170
171 /* Erratum 170 which requires a forced TLB flush for each world switch:
172 * See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
173 *
174 * All BH-G1/2 and DH-G1/2 models include a fix:
175 * Athlon X2: 0x6b 1/2
176 * 0x68 1/2
177 * Athlon 64: 0x7f 1
178 * 0x6f 2
179 * Sempron: 0x7f 1/2
180 * 0x6f 2
181 * 0x6c 2
182 * 0x7c 2
183 * Turion 64: 0x68 2
184 *
185 */
186 uint32_t u32Dummy;
187 uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
188 ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
189 u32BaseFamily= (u32Version >> 8) & 0xf;
190 u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
191 u32Model = ((u32Version >> 4) & 0xf);
192 u32Model = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
193 u32Stepping = u32Version & 0xf;
194 if ( u32Family == 0xf
195 && !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) && u32Stepping >= 1)
196 && !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) && u32Stepping >= 2))
197 {
198 Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
199 pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
200 }
201 return VINF_SUCCESS;
202}
203
204/**
205 * Does Ring-0 per VM AMD-V termination.
206 *
207 * @returns VBox status code.
208 * @param pVM The VM to operate on.
209 */
210VMMR0DECL(int) SVMR0TermVM(PVM pVM)
211{
212 if (pVM->hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
213 {
214 RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCB, false);
215 pVM->hwaccm.s.svm.pVMCB = 0;
216 pVM->hwaccm.s.svm.pVMCBPhys = 0;
217 pVM->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
218 }
219 if (pVM->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
220 {
221 RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCBHost, false);
222 pVM->hwaccm.s.svm.pVMCBHost = 0;
223 pVM->hwaccm.s.svm.pVMCBHostPhys = 0;
224 pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
225 }
226 if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
227 {
228 RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
229 pVM->hwaccm.s.svm.pIOBitmap = 0;
230 pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
231 pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
232 }
233 if (pVM->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
234 {
235 RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
236 pVM->hwaccm.s.svm.pMSRBitmap = 0;
237 pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
238 pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
239 }
240 return VINF_SUCCESS;
241}
242
243/**
244 * Sets up AMD-V for the specified VM
245 *
246 * @returns VBox status code.
247 * @param pVM The VM to operate on.
248 */
249VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
250{
251 int rc = VINF_SUCCESS;
252 SVM_VMCB *pVMCB;
253
254 AssertReturn(pVM, VERR_INVALID_PARAMETER);
255
256 Assert(pVM->hwaccm.s.svm.fSupported);
257
258 pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
259 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
260
261 /* Program the control fields. Most of them never have to be changed again. */
262 /* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
263 /* Note: CR0 & CR4 can be safely read when guest and shadow copies are identical. */
264 if (!pVM->hwaccm.s.fNestedPaging)
265 pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) | RT_BIT(3) | RT_BIT(4);
266 else
267 pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) | RT_BIT(4);
268
269 /*
270 * CR0/3/4 writes must be intercepted for obvious reasons.
271 */
272 if (!pVM->hwaccm.s.fNestedPaging)
273 pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) | RT_BIT(3) | RT_BIT(4);
274 else
275 pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) | RT_BIT(4) | RT_BIT(8);
276
277 /* Intercept all DRx reads and writes by default. Changed later on. */
278 pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
279 pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
280
281 /* Currently we don't care about DRx reads or writes. DRx registers are trashed.
282 * All breakpoints are automatically cleared when the VM exits.
283 */
284
285 pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
286#ifndef DEBUG
287 if (pVM->hwaccm.s.fNestedPaging)
288 pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
289#endif
290
291 pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
292 | SVM_CTRL1_INTERCEPT_VINTR
293 | SVM_CTRL1_INTERCEPT_NMI
294 | SVM_CTRL1_INTERCEPT_SMI
295 | SVM_CTRL1_INTERCEPT_INIT
296 | SVM_CTRL1_INTERCEPT_RDPMC
297 | SVM_CTRL1_INTERCEPT_CPUID
298 | SVM_CTRL1_INTERCEPT_RSM
299 | SVM_CTRL1_INTERCEPT_HLT
300 | SVM_CTRL1_INTERCEPT_INOUT_BITMAP
301 | SVM_CTRL1_INTERCEPT_MSR_SHADOW
302 | SVM_CTRL1_INTERCEPT_INVLPG
303 | SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
304 | SVM_CTRL1_INTERCEPT_TASK_SWITCH
305 | SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
306 | SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
307 ;
308 /* With nested paging we don't care about invlpg anymore. */
309 if (pVM->hwaccm.s.fNestedPaging)
310 pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
311
312 pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
313 | SVM_CTRL2_INTERCEPT_VMMCALL
314 | SVM_CTRL2_INTERCEPT_VMLOAD
315 | SVM_CTRL2_INTERCEPT_VMSAVE
316 | SVM_CTRL2_INTERCEPT_STGI
317 | SVM_CTRL2_INTERCEPT_CLGI
318 | SVM_CTRL2_INTERCEPT_SKINIT
319 | SVM_CTRL2_INTERCEPT_RDTSCP /* AMD only; we don't support this one */
320 | SVM_CTRL2_INTERCEPT_WBINVD
321 | SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
322 ;
323 Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
324 Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
325 Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
326
327 /* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
328 pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
329 /* Ignore the priority in the TPR; just deliver it when we tell it to. */
330 pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
331
332 /* Set IO and MSR bitmap addresses. */
333 pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
334 pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
335
336 /* No LBR virtualization. */
337 pVMCB->ctrl.u64LBRVirt = 0;
338
339 /** The ASID must start at 1; the host uses 0. */
340 pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
341
342 /** Setup the PAT msr (nested paging only) */
343 pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
344 return rc;
345}
346
347
348/**
349 * Injects an event (trap or external interrupt)
350 *
351 * @param pVM The VM to operate on.
352 * @param pVMCB SVM control block
353 * @param pCtx CPU Context
354 * @param pIntInfo SVM interrupt info
355 */
356inline void SVMR0InjectEvent(PVM pVM, SVM_VMCB *pVMCB, CPUMCTX *pCtx, SVM_EVENT* pEvent)
357{
358#ifdef VBOX_STRICT
359 if (pEvent->n.u8Vector == 0xE)
360 Log(("SVM: Inject int %d at %VGv error code=%02x CR2=%VGv intInfo=%08x\n", pEvent->n.u8Vector, pCtx->rip, pEvent->n.u32ErrorCode, pCtx->cr2, pEvent->au64[0]));
361 else
362 if (pEvent->n.u8Vector < 0x20)
363 Log(("SVM: Inject int %d at %VGv error code=%08x\n", pEvent->n.u8Vector, pCtx->rip, pEvent->n.u32ErrorCode));
364 else
365 {
366 Log(("INJ-EI: %x at %VGv\n", pEvent->n.u8Vector, pCtx->rip));
367 Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
368 Assert(pCtx->eflags.u32 & X86_EFL_IF);
369 }
370#endif
371
372 /* Set event injection state. */
373 pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
374}
375
376
377/**
378 * Checks for pending guest interrupts and injects them
379 *
380 * @returns VBox status code.
381 * @param pVM The VM to operate on.
382 * @param pVMCB SVM control block
383 * @param pCtx CPU Context
384 */
385static int SVMR0CheckPendingInterrupt(PVM pVM, SVM_VMCB *pVMCB, CPUMCTX *pCtx)
386{
387 int rc;
388
389 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
390 if (pVM->hwaccm.s.Event.fPending)
391 {
392 SVM_EVENT Event;
393
394 Log(("Reinjecting event %08x %08x at %VGv\n", pVM->hwaccm.s.Event.intInfo, pVM->hwaccm.s.Event.errCode, pCtx->rip));
395 STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntReinject);
396 Event.au64[0] = pVM->hwaccm.s.Event.intInfo;
397 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
398
399 pVM->hwaccm.s.Event.fPending = false;
400 return VINF_SUCCESS;
401 }
402
403 /* When external interrupts are pending, we should exit the VM when IF is set. */
404 if ( !TRPMHasTrap(pVM)
405 && VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC|VM_FF_INTERRUPT_PIC)))
406 {
407 if ( !(pCtx->eflags.u32 & X86_EFL_IF)
408 || VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
409 {
410 if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
411 {
412 if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
413 LogFlow(("Enable irq window exit!\n"));
414 else
415 Log(("Pending interrupt blocked at %VGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n", pCtx->rip));
416
417 /** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
418 pVMCB->ctrl.u32InterceptCtrl1 |= SVM_CTRL1_INTERCEPT_VINTR;
419 pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
420 pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
421 }
422 }
423 else
424 {
425 uint8_t u8Interrupt;
426
427 rc = PDMGetInterrupt(pVM, &u8Interrupt);
428 Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Vrc\n", u8Interrupt, u8Interrupt, rc));
429 if (VBOX_SUCCESS(rc))
430 {
431 rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
432 AssertRC(rc);
433 }
434 else
435 {
436 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
437 Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC|VM_FF_INTERRUPT_PIC)));
438 STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchGuestIrq);
439 /* Just continue */
440 }
441 }
442 }
443
444#ifdef VBOX_STRICT
445 if (TRPMHasTrap(pVM))
446 {
447 uint8_t u8Vector;
448 rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
449 AssertRC(rc);
450 }
451#endif
452
453 if ( pCtx->eflags.u32 & X86_EFL_IF
454 && (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
455 && TRPMHasTrap(pVM)
456 )
457 {
458 uint8_t u8Vector;
459 int rc;
460 TRPMEVENT enmType;
461 SVM_EVENT Event;
462 RTGCUINT u32ErrorCode;
463
464 Event.au64[0] = 0;
465
466 /* If a new event is pending, then dispatch it now. */
467 rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &u32ErrorCode, 0);
468 AssertRC(rc);
469 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
470 Assert(enmType != TRPM_SOFTWARE_INT);
471
472 /* Clear the pending trap. */
473 rc = TRPMResetTrap(pVM);
474 AssertRC(rc);
475
476 Event.n.u8Vector = u8Vector;
477 Event.n.u1Valid = 1;
478 Event.n.u32ErrorCode = u32ErrorCode;
479
480 if (enmType == TRPM_TRAP)
481 {
482 switch (u8Vector) {
483 case 8:
484 case 10:
485 case 11:
486 case 12:
487 case 13:
488 case 14:
489 case 17:
490 /* Valid error codes. */
491 Event.n.u1ErrorCodeValid = 1;
492 break;
493 default:
494 break;
495 }
496 if (u8Vector == X86_XCPT_NMI)
497 Event.n.u3Type = SVM_EVENT_NMI;
498 else
499 Event.n.u3Type = SVM_EVENT_EXCEPTION;
500 }
501 else
502 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
503
504 STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntInject);
505 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
506 } /* if (interrupts can be dispatched) */
507
508 return VINF_SUCCESS;
509}
510
511/**
512 * Save the host state
513 *
514 * @returns VBox status code.
515 * @param pVM The VM to operate on.
516 */
517VMMR0DECL(int) SVMR0SaveHostState(PVM pVM)
518{
519 /* Nothing to do here. */
520 return VINF_SUCCESS;
521}
522
523/**
524 * Loads the guest state
525 *
526 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
527 *
528 * @returns VBox status code.
529 * @param pVM The VM to operate on.
530 * @param pCtx Guest context
531 */
532VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, CPUMCTX *pCtx)
533{
534 RTGCUINTPTR val;
535 SVM_VMCB *pVMCB;
536
537 if (pVM == NULL)
538 return VERR_INVALID_PARAMETER;
539
540 /* Setup AMD SVM. */
541 Assert(pVM->hwaccm.s.svm.fSupported);
542
543 pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
544 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
545
546 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
547 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
548 {
549 SVM_WRITE_SELREG(CS, cs);
550 SVM_WRITE_SELREG(SS, ss);
551 SVM_WRITE_SELREG(DS, ds);
552 SVM_WRITE_SELREG(ES, es);
553 SVM_WRITE_SELREG(FS, fs);
554 SVM_WRITE_SELREG(GS, gs);
555 }
556
557 /* Guest CPU context: LDTR. */
558 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
559 {
560 SVM_WRITE_SELREG(LDTR, ldtr);
561 }
562
563 /* Guest CPU context: TR. */
564 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
565 {
566 SVM_WRITE_SELREG(TR, tr);
567 }
568
569 /* Guest CPU context: GDTR. */
570 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
571 {
572 pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
573 pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
574 }
575
576 /* Guest CPU context: IDTR. */
577 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
578 {
579 pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
580 pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
581 }
582
583 /*
584 * Sysenter MSRs (unconditional)
585 */
586 pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
587 pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
588 pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
589
590 /* Control registers */
591 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
592 {
593 val = pCtx->cr0;
594 if (!CPUMIsGuestFPUStateActive(pVM))
595 {
596 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
597 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
598 }
599 else
600 {
601 /** @todo check if we support the old style mess correctly. */
602 if (!(val & X86_CR0_NE))
603 {
604 Log(("Forcing X86_CR0_NE!!!\n"));
605
606 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
607 if (!pVM->hwaccm.s.fFPUOldStyleOverride)
608 {
609 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_MF);
610 pVM->hwaccm.s.fFPUOldStyleOverride = true;
611 }
612 }
613 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
614 }
615 /* Always enable caching. */
616 val &= ~(X86_CR0_CD|X86_CR0_NW);
617
618 /* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
619 /* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
620 if (!pVM->hwaccm.s.fNestedPaging)
621 {
622 val |= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
623 val |= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
624 }
625 pVMCB->guest.u64CR0 = val;
626 }
627 /* CR2 as well */
628 pVMCB->guest.u64CR2 = pCtx->cr2;
629
630 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
631 {
632 /* Save our shadow CR3 register. */
633 if (pVM->hwaccm.s.fNestedPaging)
634 {
635 pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVM, PGMGetHostMode(pVM));
636 Assert(pVMCB->ctrl.u64NestedPagingCR3);
637 pVMCB->guest.u64CR3 = pCtx->cr3;
638 }
639 else
640 {
641 pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVM);
642 Assert(pVMCB->guest.u64CR3);
643 }
644 }
645
646 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
647 {
648 val = pCtx->cr4;
649 if (!pVM->hwaccm.s.fNestedPaging)
650 {
651 switch(pVM->hwaccm.s.enmShadowMode)
652 {
653 case PGMMODE_REAL:
654 case PGMMODE_PROTECTED: /* Protected mode, no paging. */
655 AssertFailed();
656 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
657
658 case PGMMODE_32_BIT: /* 32-bit paging. */
659 break;
660
661 case PGMMODE_PAE: /* PAE paging. */
662 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
663 /** @todo use normal 32 bits paging */
664 val |= X86_CR4_PAE;
665 break;
666
667 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
668 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
669#ifdef VBOX_ENABLE_64_BITS_GUESTS
670 break;
671#else
672 AssertFailed();
673 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
674#endif
675
676 default: /* shut up gcc */
677 AssertFailed();
678 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
679 }
680 }
681 pVMCB->guest.u64CR4 = val;
682 }
683
684 /* Debug registers. */
685 if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
686 {
687 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
688 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
689
690 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
691 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
692 pCtx->dr[7] |= 0x400; /* must be one */
693
694 pVMCB->guest.u64DR7 = pCtx->dr[7];
695 pVMCB->guest.u64DR6 = pCtx->dr[6];
696
697 /* Sync the debug state now if any breakpoint is armed. */
698 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
699 && !CPUMIsGuestDebugStateActive(pVM)
700 && !DBGFIsStepping(pVM))
701 {
702 STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxArmed);
703
704 /* Disable drx move intercepts. */
705 pVMCB->ctrl.u16InterceptRdDRx = 0;
706 pVMCB->ctrl.u16InterceptWrDRx = 0;
707
708 /* Save the host and load the guest debug state. */
709 int rc = CPUMR0LoadGuestDebugState(pVM, pCtx, false /* exclude DR6 */);
710 AssertRC(rc);
711 }
712 }
713
714 /* EIP, ESP and EFLAGS */
715 pVMCB->guest.u64RIP = pCtx->rip;
716 pVMCB->guest.u64RSP = pCtx->rsp;
717 pVMCB->guest.u64RFlags = pCtx->eflags.u32;
718
719 /* Set CPL */
720 pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
721
722 /* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
723 pVMCB->guest.u64RAX = pCtx->rax;
724
725 /* vmrun will fail without MSR_K6_EFER_SVME. */
726 pVMCB->guest.u64EFER = pCtx->msrEFER | MSR_K6_EFER_SVME;
727
728 /* 64 bits guest mode? */
729 if (pCtx->msrEFER & MSR_K6_EFER_LMA)
730 {
731#if !defined(VBOX_WITH_64_BITS_GUESTS) || HC_ARCH_BITS != 64
732 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
733#else
734 pVM->hwaccm.s.svm.pfnVMRun = SVMVMRun64;
735#endif
736 /* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
737 pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
738 pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
739 }
740 else
741 {
742 /* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
743 pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
744
745 pVM->hwaccm.s.svm.pfnVMRun = SVMVMRun;
746 }
747
748 /* TSC offset. */
749 if (TMCpuTickCanUseRealTSC(pVM, &pVMCB->ctrl.u64TSCOffset))
750 {
751 pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
752 STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCOffset);
753 }
754 else
755 {
756 pVMCB->ctrl.u32InterceptCtrl1 |= SVM_CTRL1_INTERCEPT_RDTSC;
757 STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCIntercept);
758 }
759
760 /* Sync the various msrs for 64 bits mode. */
761 pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
762 pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
763 pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
764 pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
765 pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
766
767#ifdef DEBUG
768 /* Intercept X86_XCPT_DB if stepping is enabled */
769 if (DBGFIsStepping(pVM))
770 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_DB);
771 else
772 pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
773#endif
774
775 /* Done. */
776 pVM->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
777
778 return VINF_SUCCESS;
779}
780
781
782/**
783 * Runs guest code in an SVM VM.
784 *
785 * @todo This can be much more efficient, when we only sync that which has actually changed. (this is the first attempt only)
786 *
787 * @returns VBox status code.
788 * @param pVM The VM to operate on.
789 * @param pCtx Guest context
790 */
791VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, CPUMCTX *pCtx)
792{
793 int rc = VINF_SUCCESS;
794 uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
795 SVM_VMCB *pVMCB;
796 bool fSyncTPR = false;
797 unsigned cResume = 0;
798 uint8_t u8LastVTPR;
799 PHWACCM_CPUINFO pCpu = 0;
800#ifdef VBOX_STRICT
801 RTCPUID idCpuCheck;
802#endif
803
804 STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatEntry, x);
805
806 pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
807 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
808
809 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
810 */
811ResumeExecution:
812 /* Safety precaution; looping for too long here can have a very bad effect on the host */
813 if (++cResume > HWACCM_MAX_RESUME_LOOPS)
814 {
815 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitMaxResume);
816 rc = VINF_EM_RAW_INTERRUPT;
817 goto end;
818 }
819
820 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
821 if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
822 {
823 Log(("VM_FF_INHIBIT_INTERRUPTS at %VGv successor %VGv\n", pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
824 if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
825 {
826 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
827 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
828 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
829 * break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
830 */
831 VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
832 /* Irq inhibition is no longer active; clear the corresponding SVM state. */
833 pVMCB->ctrl.u64IntShadow = 0;
834 }
835 }
836 else
837 {
838 /* Irq inhibition is no longer active; clear the corresponding SVM state. */
839 pVMCB->ctrl.u64IntShadow = 0;
840 }
841
842 /* Check for pending actions that force us to go back to ring 3. */
843#ifdef DEBUG
844 /* Intercept X86_XCPT_DB if stepping is enabled */
845 if (!DBGFIsStepping(pVM))
846#endif
847 {
848 if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 | VM_FF_TIMER))
849 {
850 VM_FF_CLEAR(pVM, VM_FF_TO_R3);
851 STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchToR3);
852 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
853 rc = VINF_EM_RAW_TO_R3;
854 goto end;
855 }
856 }
857
858 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
859 if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
860 {
861 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
862 rc = VINF_EM_PENDING_REQUEST;
863 goto end;
864 }
865
866 /* When external interrupts are pending, we should exit the VM when IF is set. */
867 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
868 rc = SVMR0CheckPendingInterrupt(pVM, pVMCB, pCtx);
869 if (VBOX_FAILURE(rc))
870 {
871 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
872 goto end;
873 }
874
875 /* TPR caching using CR8 is only available in 64 bits mode */
876 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
877 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!!!!! */
878 if (pCtx->msrEFER & MSR_K6_EFER_LMA)
879 {
880 bool fPending;
881
882 /* TPR caching in CR8 */
883 int rc = PDMApicGetTPR(pVM, &u8LastVTPR, &fPending);
884 AssertRC(rc);
885 pVMCB->ctrl.IntCtrl.n.u8VTPR = u8LastVTPR;
886
887 if (fPending)
888 {
889 /* A TPR change could activate a pending interrupt, so catch cr8 writes. */
890 pVMCB->ctrl.u16InterceptWrCRx |= RT_BIT(8);
891 }
892 else
893 /* No interrupts are pending, so we don't need to be explicitely notified.
894 * There are enough world switches for detecting pending interrupts.
895 */
896 pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
897
898 fSyncTPR = !fPending;
899 }
900
901 /* All done! Let's start VM execution. */
902 STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatInGC, x);
903
904 /* Enable nested paging if necessary (disabled each time after #VMEXIT). */
905 pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
906
907#ifdef LOG_ENABLED
908 pCpu = HWACCMR0GetCurrentCpu();
909 if ( pVM->hwaccm.s.idLastCpu != pCpu->idCpu
910 || pVM->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
911 {
912 if (pVM->hwaccm.s.idLastCpu != pCpu->idCpu)
913 Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVM->hwaccm.s.idLastCpu, pCpu->idCpu));
914 else
915 Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVM->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
916 }
917 if (pCpu->fFlushTLB)
918 Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
919#endif
920
921 /*
922 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
923 * (until the actual world switch)
924 */
925
926#ifdef VBOX_STRICT
927 idCpuCheck = RTMpCpuId();
928#endif
929
930 /* Load the guest state; *must* be here as it sets up the shadow cr0 for lazy fpu syncing! */
931 rc = SVMR0LoadGuestState(pVM, pCtx);
932 if (rc != VINF_SUCCESS)
933 {
934 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
935 goto end;
936 }
937
938 pCpu = HWACCMR0GetCurrentCpu();
939 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
940 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
941 if ( pVM->hwaccm.s.idLastCpu != pCpu->idCpu
942 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
943 || pVM->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
944 {
945 /* Force a TLB flush on VM entry. */
946 pVM->hwaccm.s.fForceTLBFlush = true;
947 }
948 else
949 Assert(!pCpu->fFlushTLB || pVM->hwaccm.s.svm.fAlwaysFlushTLB);
950
951 pVM->hwaccm.s.idLastCpu = pCpu->idCpu;
952
953 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
954 if ( pVM->hwaccm.s.fForceTLBFlush
955 && !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
956 {
957 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
958 || pCpu->fFlushTLB)
959 {
960 pCpu->fFlushTLB = false;
961 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
962 pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
963 pCpu->cTLBFlushes++;
964 }
965 else
966 STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushASID);
967
968 pVM->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
969 pVM->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
970 }
971 else
972 {
973 Assert(!pCpu->fFlushTLB || pVM->hwaccm.s.svm.fAlwaysFlushTLB);
974
975 /* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
976 if (!pCpu->uCurrentASID || !pVM->hwaccm.s.uCurrentASID)
977 pVM->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
978
979 Assert(!pVM->hwaccm.s.svm.fAlwaysFlushTLB || pVM->hwaccm.s.fForceTLBFlush);
980 pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVM->hwaccm.s.fForceTLBFlush;
981 }
982 AssertMsg(pVM->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVM->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
983 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
984 AssertMsg(pVM->hwaccm.s.uCurrentASID >= 1 && pVM->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVM->hwaccm.s.uCurrentASID));
985 pVMCB->ctrl.TLBCtrl.n.u32ASID = pVM->hwaccm.s.uCurrentASID;
986
987#ifdef VBOX_WITH_STATISTICS
988 if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
989 STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBWorldSwitch);
990 else
991 STAM_COUNTER_INC(&pVM->hwaccm.s.StatNoFlushTLBWorldSwitch);
992#endif
993
994 /* In case we execute a goto ResumeExecution later on. */
995 pVM->hwaccm.s.svm.fResumeVM = true;
996 pVM->hwaccm.s.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
997
998 Assert(sizeof(pVM->hwaccm.s.svm.pVMCBPhys) == 8);
999 Assert(pVMCB->ctrl.u32InterceptCtrl2 == ( SVM_CTRL2_INTERCEPT_VMRUN /* required */
1000 | SVM_CTRL2_INTERCEPT_VMMCALL
1001 | SVM_CTRL2_INTERCEPT_VMLOAD
1002 | SVM_CTRL2_INTERCEPT_VMSAVE
1003 | SVM_CTRL2_INTERCEPT_STGI
1004 | SVM_CTRL2_INTERCEPT_CLGI
1005 | SVM_CTRL2_INTERCEPT_SKINIT
1006 | SVM_CTRL2_INTERCEPT_RDTSCP /* AMD only; we don't support this one */
1007 | SVM_CTRL2_INTERCEPT_WBINVD
1008 | SVM_CTRL2_INTERCEPT_MWAIT_UNCOND /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
1009 ));
1010 Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1011 Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
1012 Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
1013 Assert(pVMCB->ctrl.u64LBRVirt == 0);
1014
1015#ifdef VBOX_STRICT
1016 Assert(idCpuCheck == RTMpCpuId());
1017#endif
1018 TMNotifyStartOfExecution(pVM);
1019 pVM->hwaccm.s.svm.pfnVMRun(pVM->hwaccm.s.svm.pVMCBHostPhys, pVM->hwaccm.s.svm.pVMCBPhys, pCtx);
1020 TMNotifyEndOfExecution(pVM);
1021 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatInGC, x);
1022
1023 /*
1024 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1025 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1026 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1027 */
1028
1029 STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatExit, x);
1030
1031 /* Reason for the VM exit */
1032 exitCode = pVMCB->ctrl.u64ExitCode;
1033
1034 if (exitCode == (uint64_t)SVM_EXIT_INVALID) /* Invalid guest state. */
1035 {
1036 HWACCMDumpRegs(pVM, pCtx);
1037#ifdef DEBUG
1038 Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1039 Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1040 Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1041 Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1042 Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1043 Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1044 Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1045 Log(("ctrl.u64IOPMPhysAddr %VX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1046 Log(("ctrl.u64MSRPMPhysAddr %VX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1047 Log(("ctrl.u64TSCOffset %VX64\n", pVMCB->ctrl.u64TSCOffset));
1048
1049 Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1050 Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
1051 Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
1052 Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1053
1054 Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1055 Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1056 Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1057 Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1058 Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1059 Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1060 Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1061 Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1062 Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1063 Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1064
1065 Log(("ctrl.u64IntShadow %VX64\n", pVMCB->ctrl.u64IntShadow));
1066 Log(("ctrl.u64ExitCode %VX64\n", pVMCB->ctrl.u64ExitCode));
1067 Log(("ctrl.u64ExitInfo1 %VX64\n", pVMCB->ctrl.u64ExitInfo1));
1068 Log(("ctrl.u64ExitInfo2 %VX64\n", pVMCB->ctrl.u64ExitInfo2));
1069 Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1070 Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1071 Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1072 Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1073 Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1074 Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1075 Log(("ctrl.NestedPaging %VX64\n", pVMCB->ctrl.NestedPaging.au64));
1076 Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1077 Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1078 Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1079 Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1080 Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1081 Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1082
1083 Log(("ctrl.u64NestedPagingCR3 %VX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1084 Log(("ctrl.u64LBRVirt %VX64\n", pVMCB->ctrl.u64LBRVirt));
1085
1086 Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1087 Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1088 Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1089 Log(("guest.CS.u64Base %VX64\n", pVMCB->guest.CS.u64Base));
1090 Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1091 Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1092 Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1093 Log(("guest.DS.u64Base %VX64\n", pVMCB->guest.DS.u64Base));
1094 Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1095 Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1096 Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1097 Log(("guest.ES.u64Base %VX64\n", pVMCB->guest.ES.u64Base));
1098 Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1099 Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1100 Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1101 Log(("guest.FS.u64Base %VX64\n", pVMCB->guest.FS.u64Base));
1102 Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1103 Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1104 Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1105 Log(("guest.GS.u64Base %VX64\n", pVMCB->guest.GS.u64Base));
1106
1107 Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1108 Log(("guest.GDTR.u64Base %VX64\n", pVMCB->guest.GDTR.u64Base));
1109
1110 Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1111 Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1112 Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1113 Log(("guest.LDTR.u64Base %VX64\n", pVMCB->guest.LDTR.u64Base));
1114
1115 Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1116 Log(("guest.IDTR.u64Base %VX64\n", pVMCB->guest.IDTR.u64Base));
1117
1118 Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1119 Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1120 Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1121 Log(("guest.TR.u64Base %VX64\n", pVMCB->guest.TR.u64Base));
1122
1123 Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1124 Log(("guest.u64CR0 %VX64\n", pVMCB->guest.u64CR0));
1125 Log(("guest.u64CR2 %VX64\n", pVMCB->guest.u64CR2));
1126 Log(("guest.u64CR3 %VX64\n", pVMCB->guest.u64CR3));
1127 Log(("guest.u64CR4 %VX64\n", pVMCB->guest.u64CR4));
1128 Log(("guest.u64DR6 %VX64\n", pVMCB->guest.u64DR6));
1129 Log(("guest.u64DR7 %VX64\n", pVMCB->guest.u64DR7));
1130
1131 Log(("guest.u64RIP %VX64\n", pVMCB->guest.u64RIP));
1132 Log(("guest.u64RSP %VX64\n", pVMCB->guest.u64RSP));
1133 Log(("guest.u64RAX %VX64\n", pVMCB->guest.u64RAX));
1134 Log(("guest.u64RFlags %VX64\n", pVMCB->guest.u64RFlags));
1135
1136 Log(("guest.u64SysEnterCS %VX64\n", pVMCB->guest.u64SysEnterCS));
1137 Log(("guest.u64SysEnterEIP %VX64\n", pVMCB->guest.u64SysEnterEIP));
1138 Log(("guest.u64SysEnterESP %VX64\n", pVMCB->guest.u64SysEnterESP));
1139
1140 Log(("guest.u64EFER %VX64\n", pVMCB->guest.u64EFER));
1141 Log(("guest.u64STAR %VX64\n", pVMCB->guest.u64STAR));
1142 Log(("guest.u64LSTAR %VX64\n", pVMCB->guest.u64LSTAR));
1143 Log(("guest.u64CSTAR %VX64\n", pVMCB->guest.u64CSTAR));
1144 Log(("guest.u64SFMASK %VX64\n", pVMCB->guest.u64SFMASK));
1145 Log(("guest.u64KernelGSBase %VX64\n", pVMCB->guest.u64KernelGSBase));
1146 Log(("guest.u64GPAT %VX64\n", pVMCB->guest.u64GPAT));
1147 Log(("guest.u64DBGCTL %VX64\n", pVMCB->guest.u64DBGCTL));
1148 Log(("guest.u64BR_FROM %VX64\n", pVMCB->guest.u64BR_FROM));
1149 Log(("guest.u64BR_TO %VX64\n", pVMCB->guest.u64BR_TO));
1150 Log(("guest.u64LASTEXCPFROM %VX64\n", pVMCB->guest.u64LASTEXCPFROM));
1151 Log(("guest.u64LASTEXCPTO %VX64\n", pVMCB->guest.u64LASTEXCPTO));
1152
1153#endif
1154 rc = VERR_SVM_UNABLE_TO_START_VM;
1155 goto end;
1156 }
1157
1158 /* Let's first sync back eip, esp, and eflags. */
1159 pCtx->rip = pVMCB->guest.u64RIP;
1160 pCtx->rsp = pVMCB->guest.u64RSP;
1161 pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1162 /* eax is saved/restore across the vmrun instruction */
1163 pCtx->rax = pVMCB->guest.u64RAX;
1164
1165 pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1166
1167 /* Can be updated behind our back in the nested paging case. */
1168 pCtx->cr2 = pVMCB->guest.u64CR2;
1169
1170 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1171 SVM_READ_SELREG(SS, ss);
1172 SVM_READ_SELREG(CS, cs);
1173 SVM_READ_SELREG(DS, ds);
1174 SVM_READ_SELREG(ES, es);
1175 SVM_READ_SELREG(FS, fs);
1176 SVM_READ_SELREG(GS, gs);
1177
1178 /*
1179 * System MSRs
1180 */
1181 pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1182 pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1183 pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1184
1185 /* Remaining guest CPU context: TR, IDTR, GDTR, LDTR; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1186 SVM_READ_SELREG(LDTR, ldtr);
1187 SVM_READ_SELREG(TR, tr);
1188
1189 pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1190 pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1191
1192 pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1193 pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1194
1195 /* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1196 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1197 if ( pVM->hwaccm.s.fNestedPaging
1198 && pCtx->cr3 != pVMCB->guest.u64CR3)
1199 {
1200 CPUMSetGuestCR3(pVM, pVMCB->guest.u64CR3);
1201 PGMUpdateCR3(pVM, pVMCB->guest.u64CR3);
1202 }
1203
1204 /* Note! NOW IT'S SAFE FOR LOGGING! */
1205
1206 /* Take care of instruction fusing (sti, mov ss) (see 15.20.5 Interrupt Shadows) */
1207 if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1208 {
1209 Log(("uInterruptState %x eip=%VGv\n", pVMCB->ctrl.u64IntShadow, pCtx->rip));
1210 EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1211 }
1212 else
1213 VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1214
1215 Log2(("exitCode = %x\n", exitCode));
1216
1217 /* Sync back DR6 as it could have been changed by hitting breakpoints. */
1218 pCtx->dr[6] = pVMCB->guest.u64DR6;
1219 /* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1220 pCtx->dr[7] = pVMCB->guest.u64DR7;
1221
1222 /* Check if an injected event was interrupted prematurely. */
1223 pVM->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1224 if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1225 && pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1226 {
1227 Log(("Pending inject %VX64 at %VGv exit=%08x\n", pVM->hwaccm.s.Event.intInfo, pCtx->rip, exitCode));
1228
1229#ifdef LOG_ENABLED
1230 SVM_EVENT Event;
1231 Event.au64[0] = pVM->hwaccm.s.Event.intInfo;
1232
1233 if ( exitCode == SVM_EXIT_EXCEPTION_E
1234 && Event.n.u8Vector == 0xE)
1235 {
1236 Log(("Double fault!\n"));
1237 }
1238#endif
1239
1240 pVM->hwaccm.s.Event.fPending = true;
1241 /* Error code present? (redundant) */
1242 if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1243 {
1244 pVM->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1245 }
1246 else
1247 pVM->hwaccm.s.Event.errCode = 0;
1248 }
1249#ifdef VBOX_WITH_STATISTICS
1250 if (exitCode == SVM_EXIT_NPF)
1251 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitReasonNPF);
1252 else
1253 STAM_COUNTER_INC(&pVM->hwaccm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1254#endif
1255
1256 if (fSyncTPR)
1257 {
1258 rc = PDMApicSetTPR(pVM, pVMCB->ctrl.IntCtrl.n.u8VTPR);
1259 AssertRC(rc);
1260 }
1261
1262 /* Deal with the reason of the VM-exit. */
1263 switch (exitCode)
1264 {
1265 case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1266 case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1267 case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1268 case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1269 case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1270 case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1271 case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1272 case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1273 {
1274 /* Pending trap. */
1275 SVM_EVENT Event;
1276 uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1277
1278 Log2(("Hardware/software interrupt %d\n", vector));
1279 switch (vector)
1280 {
1281 case X86_XCPT_DB:
1282 {
1283 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestDB);
1284
1285 /* Note that we don't support guest and host-initiated debugging at the same time. */
1286 Assert(DBGFIsStepping(pVM));
1287
1288 rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1289 if (rc == VINF_EM_RAW_GUEST_TRAP)
1290 {
1291 Log(("Trap %x (debug) at %VGv\n", vector, pCtx->rip));
1292
1293 /* Reinject the exception. */
1294 Event.au64[0] = 0;
1295 Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1296 Event.n.u1Valid = 1;
1297 Event.n.u8Vector = X86_XCPT_DB;
1298
1299 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1300
1301 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1302 goto ResumeExecution;
1303 }
1304 /* Return to ring 3 to deal with the debug exit code. */
1305 break;
1306 }
1307
1308 case X86_XCPT_NM:
1309 {
1310 Log(("#NM fault at %VGv\n", pCtx->rip));
1311
1312 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1313 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1314 rc = CPUMR0LoadGuestFPU(pVM, pCtx);
1315 if (rc == VINF_SUCCESS)
1316 {
1317 Assert(CPUMIsGuestFPUStateActive(pVM));
1318 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowNM);
1319
1320 /* Continue execution. */
1321 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1322 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
1323
1324 goto ResumeExecution;
1325 }
1326
1327 Log(("Forward #NM fault to the guest\n"));
1328 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNM);
1329
1330 Event.au64[0] = 0;
1331 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1332 Event.n.u1Valid = 1;
1333 Event.n.u8Vector = X86_XCPT_NM;
1334
1335 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1336 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1337 goto ResumeExecution;
1338 }
1339
1340 case X86_XCPT_PF: /* Page fault */
1341 {
1342 uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1343 RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1344
1345#ifdef DEBUG
1346 if (pVM->hwaccm.s.fNestedPaging)
1347 { /* A genuine pagefault.
1348 * Forward the trap to the guest by injecting the exception and resuming execution.
1349 */
1350 Log(("Guest page fault at %VGv cr2=%VGv error code %x rsp=%VGv\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1351 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1352
1353 /* Now we must update CR2. */
1354 pCtx->cr2 = uFaultAddress;
1355
1356 Event.au64[0] = 0;
1357 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1358 Event.n.u1Valid = 1;
1359 Event.n.u8Vector = X86_XCPT_PF;
1360 Event.n.u1ErrorCodeValid = 1;
1361 Event.n.u32ErrorCode = errCode;
1362
1363 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1364
1365 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1366 goto ResumeExecution;
1367 }
1368#endif
1369 Assert(!pVM->hwaccm.s.fNestedPaging);
1370
1371 Log2(("Page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, uFaultAddress, errCode));
1372 /* Exit qualification contains the linear address of the page fault. */
1373 TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1374 TRPMSetErrorCode(pVM, errCode);
1375 TRPMSetFaultAddress(pVM, uFaultAddress);
1376
1377 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1378 rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1379 Log2(("PGMTrap0eHandler %VGv returned %Vrc\n", pCtx->rip, rc));
1380 if (rc == VINF_SUCCESS)
1381 { /* We've successfully synced our shadow pages, so let's just continue execution. */
1382 Log2(("Shadow page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, uFaultAddress, errCode));
1383 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1384
1385 TRPMResetTrap(pVM);
1386
1387 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1388 goto ResumeExecution;
1389 }
1390 else
1391 if (rc == VINF_EM_RAW_GUEST_TRAP)
1392 { /* A genuine pagefault.
1393 * Forward the trap to the guest by injecting the exception and resuming execution.
1394 */
1395 Log2(("Forward page fault to the guest\n"));
1396 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1397 /* The error code might have been changed. */
1398 errCode = TRPMGetErrorCode(pVM);
1399
1400 TRPMResetTrap(pVM);
1401
1402 /* Now we must update CR2. */
1403 pCtx->cr2 = uFaultAddress;
1404
1405 Event.au64[0] = 0;
1406 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1407 Event.n.u1Valid = 1;
1408 Event.n.u8Vector = X86_XCPT_PF;
1409 Event.n.u1ErrorCodeValid = 1;
1410 Event.n.u32ErrorCode = errCode;
1411
1412 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1413
1414 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1415 goto ResumeExecution;
1416 }
1417#ifdef VBOX_STRICT
1418 if (rc != VINF_EM_RAW_EMULATE_INSTR)
1419 LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1420#endif
1421 /* Need to go back to the recompiler to emulate the instruction. */
1422 TRPMResetTrap(pVM);
1423 break;
1424 }
1425
1426 case X86_XCPT_MF: /* Floating point exception. */
1427 {
1428 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestMF);
1429 if (!(pCtx->cr0 & X86_CR0_NE))
1430 {
1431 /* old style FPU error reporting needs some extra work. */
1432 /** @todo don't fall back to the recompiler, but do it manually. */
1433 rc = VINF_EM_RAW_EMULATE_INSTR;
1434 break;
1435 }
1436 Log(("Trap %x at %VGv\n", vector, pCtx->rip));
1437
1438 Event.au64[0] = 0;
1439 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1440 Event.n.u1Valid = 1;
1441 Event.n.u8Vector = X86_XCPT_MF;
1442
1443 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1444
1445 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1446 goto ResumeExecution;
1447 }
1448
1449#ifdef VBOX_STRICT
1450 case X86_XCPT_GP: /* General protection failure exception.*/
1451 case X86_XCPT_UD: /* Unknown opcode exception. */
1452 case X86_XCPT_DE: /* Divide error. */
1453 case X86_XCPT_SS: /* Stack segment exception. */
1454 case X86_XCPT_NP: /* Segment not present exception. */
1455 {
1456 Event.au64[0] = 0;
1457 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1458 Event.n.u1Valid = 1;
1459 Event.n.u8Vector = vector;
1460
1461 switch(vector)
1462 {
1463 case X86_XCPT_GP:
1464 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestGP);
1465 Event.n.u1ErrorCodeValid = 1;
1466 Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1467 break;
1468 case X86_XCPT_DE:
1469 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestDE);
1470 break;
1471 case X86_XCPT_UD:
1472 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestUD);
1473 break;
1474 case X86_XCPT_SS:
1475 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestSS);
1476 Event.n.u1ErrorCodeValid = 1;
1477 Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1478 break;
1479 case X86_XCPT_NP:
1480 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNP);
1481 Event.n.u1ErrorCodeValid = 1;
1482 Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1483 break;
1484 }
1485 Log(("Trap %x at %VGv esi=%x\n", vector, pCtx->rip, pCtx->esi));
1486 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1487
1488 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1489 goto ResumeExecution;
1490 }
1491#endif
1492 default:
1493 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1494 rc = VERR_EM_INTERNAL_ERROR;
1495 break;
1496
1497 } /* switch (vector) */
1498 break;
1499 }
1500
1501 case SVM_EXIT_NPF:
1502 {
1503 /* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1504 uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1505 RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1506
1507 Assert(pVM->hwaccm.s.fNestedPaging);
1508 Log(("Nested page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, uFaultAddress, errCode));
1509 /* Exit qualification contains the linear address of the page fault. */
1510 TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1511 TRPMSetErrorCode(pVM, errCode);
1512 TRPMSetFaultAddress(pVM, uFaultAddress);
1513
1514 /* Handle the pagefault trap for the nested shadow table. */
1515 rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMGetHostMode(pVM), errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1516 Log2(("PGMR0Trap0eHandlerNestedPaging %VGv returned %Vrc\n", pCtx->rip, rc));
1517 if (rc == VINF_SUCCESS)
1518 { /* We've successfully synced our shadow pages, so let's just continue execution. */
1519 Log2(("Shadow page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, uFaultAddress, errCode));
1520 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1521
1522 TRPMResetTrap(pVM);
1523
1524 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1525 goto ResumeExecution;
1526 }
1527
1528#ifdef VBOX_STRICT
1529 if (rc != VINF_EM_RAW_EMULATE_INSTR)
1530 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1531#endif
1532 /* Need to go back to the recompiler to emulate the instruction. */
1533 TRPMResetTrap(pVM);
1534 break;
1535 }
1536
1537 case SVM_EXIT_VINTR:
1538 /* A virtual interrupt is about to be delivered, which means IF=1. */
1539 Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1540 pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1541 pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1542 goto ResumeExecution;
1543
1544 case SVM_EXIT_FERR_FREEZE:
1545 case SVM_EXIT_INTR:
1546 case SVM_EXIT_NMI:
1547 case SVM_EXIT_SMI:
1548 case SVM_EXIT_INIT:
1549 /* External interrupt; leave to allow it to be dispatched again. */
1550 rc = VINF_EM_RAW_INTERRUPT;
1551 break;
1552
1553 case SVM_EXIT_WBINVD:
1554 case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1555 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvd);
1556 /* Skip instruction and continue directly. */
1557 pCtx->rip += 2; /* Note! hardcoded opcode size! */
1558 /* Continue execution.*/
1559 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1560 goto ResumeExecution;
1561
1562 case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1563 {
1564 Log2(("SVM: Cpuid at %VGv for %x\n", pCtx->rip, pCtx->eax));
1565 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCpuid);
1566 rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
1567 if (rc == VINF_SUCCESS)
1568 {
1569 /* Update EIP and continue execution. */
1570 pCtx->rip += 2; /* Note! hardcoded opcode size! */
1571 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1572 goto ResumeExecution;
1573 }
1574 AssertMsgFailed(("EMU: cpuid failed with %Vrc\n", rc));
1575 rc = VINF_EM_RAW_EMULATE_INSTR;
1576 break;
1577 }
1578
1579 case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1580 {
1581 Log2(("SVM: Rdtsc\n"));
1582 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitRdtsc);
1583 rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
1584 if (rc == VINF_SUCCESS)
1585 {
1586 /* Update EIP and continue execution. */
1587 pCtx->rip += 2; /* Note! hardcoded opcode size! */
1588 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1589 goto ResumeExecution;
1590 }
1591 AssertMsgFailed(("EMU: rdtsc failed with %Vrc\n", rc));
1592 rc = VINF_EM_RAW_EMULATE_INSTR;
1593 break;
1594 }
1595
1596 case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1597 {
1598 Log2(("SVM: invlpg\n"));
1599 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvpg);
1600
1601 Assert(!pVM->hwaccm.s.fNestedPaging);
1602
1603 /* Truly a pita. Why can't SVM give the same information as VT-x? */
1604 rc = SVMR0InterpretInvpg(pVM, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1605 if (rc == VINF_SUCCESS)
1606 {
1607 STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPageInvlpg);
1608 goto ResumeExecution; /* eip already updated */
1609 }
1610 break;
1611 }
1612
1613 case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1614 case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1615 case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1616 case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1617 {
1618 uint32_t cbSize;
1619
1620 Log2(("SVM: %VGv mov cr%d, \n", pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1621 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxWrite);
1622 rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1623
1624 switch (exitCode - SVM_EXIT_WRITE_CR0)
1625 {
1626 case 0:
1627 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
1628 break;
1629 case 2:
1630 break;
1631 case 3:
1632 Assert(!pVM->hwaccm.s.fNestedPaging);
1633 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
1634 break;
1635 case 4:
1636 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
1637 break;
1638 case 8:
1639 break;
1640 default:
1641 AssertFailed();
1642 }
1643 /* Check if a sync operation is pending. */
1644 if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1645 && VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
1646 {
1647 rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
1648 AssertRC(rc);
1649
1650 STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBCRxChange);
1651
1652 /* Must be set by PGMSyncCR3 */
1653 Assert(PGMGetGuestMode(pVM) <= PGMMODE_PROTECTED || pVM->hwaccm.s.fForceTLBFlush);
1654 }
1655 if (rc == VINF_SUCCESS)
1656 {
1657 /* EIP has been updated already. */
1658
1659 /* Only resume if successful. */
1660 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1661 goto ResumeExecution;
1662 }
1663 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
1664 break;
1665 }
1666
1667 case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1668 case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1669 case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1670 case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1671 {
1672 uint32_t cbSize;
1673
1674 Log2(("SVM: %VGv mov x, cr%d\n", pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1675 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxRead);
1676 rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1677 if (rc == VINF_SUCCESS)
1678 {
1679 /* EIP has been updated already. */
1680
1681 /* Only resume if successful. */
1682 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1683 goto ResumeExecution;
1684 }
1685 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
1686 break;
1687 }
1688
1689 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1690 case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1691 case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1692 case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1693 {
1694 uint32_t cbSize;
1695
1696 Log2(("SVM: %VGv mov dr%d, x\n", pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1697 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
1698
1699 if (!DBGFIsStepping(pVM))
1700 {
1701 STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxContextSwitch);
1702
1703 /* Disable drx move intercepts. */
1704 pVMCB->ctrl.u16InterceptRdDRx = 0;
1705 pVMCB->ctrl.u16InterceptWrDRx = 0;
1706
1707 /* Save the host and load the guest debug state. */
1708 rc = CPUMR0LoadGuestDebugState(pVM, pCtx, false /* exclude DR6 */);
1709 AssertRC(rc);
1710
1711 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1712 goto ResumeExecution;
1713 }
1714
1715 rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1716 if (rc == VINF_SUCCESS)
1717 {
1718 /* EIP has been updated already. */
1719 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
1720
1721 /* Only resume if successful. */
1722 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1723 goto ResumeExecution;
1724 }
1725 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
1726 break;
1727 }
1728
1729 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
1730 case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
1731 case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
1732 case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
1733 {
1734 uint32_t cbSize;
1735
1736 Log2(("SVM: %VGv mov dr%d, x\n", pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
1737 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
1738
1739 if (!DBGFIsStepping(pVM))
1740 {
1741 STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxContextSwitch);
1742
1743 /* Disable drx move intercepts. */
1744 pVMCB->ctrl.u16InterceptRdDRx = 0;
1745 pVMCB->ctrl.u16InterceptWrDRx = 0;
1746
1747 /* Save the host and load the guest debug state. */
1748 rc = CPUMR0LoadGuestDebugState(pVM, pCtx, false /* exclude DR6 */);
1749 AssertRC(rc);
1750
1751 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1752 goto ResumeExecution;
1753 }
1754
1755 rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1756 if (rc == VINF_SUCCESS)
1757 {
1758 /* EIP has been updated already. */
1759
1760 /* Only resume if successful. */
1761 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1762 goto ResumeExecution;
1763 }
1764 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
1765 break;
1766 }
1767
1768 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
1769 case SVM_EXIT_IOIO: /* I/O instruction. */
1770 {
1771 SVM_IOIO_EXIT IoExitInfo;
1772 uint32_t uIOSize, uAndVal;
1773
1774 IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
1775
1776 /** @todo could use a lookup table here */
1777 if (IoExitInfo.n.u1OP8)
1778 {
1779 uIOSize = 1;
1780 uAndVal = 0xff;
1781 }
1782 else
1783 if (IoExitInfo.n.u1OP16)
1784 {
1785 uIOSize = 2;
1786 uAndVal = 0xffff;
1787 }
1788 else
1789 if (IoExitInfo.n.u1OP32)
1790 {
1791 uIOSize = 4;
1792 uAndVal = 0xffffffff;
1793 }
1794 else
1795 {
1796 AssertFailed(); /* should be fatal. */
1797 rc = VINF_EM_RAW_EMULATE_INSTR;
1798 break;
1799 }
1800
1801 if (IoExitInfo.n.u1STR)
1802 {
1803 /* ins/outs */
1804 uint32_t prefix = 0;
1805 if (IoExitInfo.n.u1REP)
1806 prefix |= PREFIX_REP;
1807
1808 if (IoExitInfo.n.u1Type == 0)
1809 {
1810 Log2(("IOMInterpretOUTSEx %VGv %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1811 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringWrite);
1812 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, prefix, uIOSize);
1813 }
1814 else
1815 {
1816 Log2(("IOMInterpretINSEx %VGv %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1817 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringRead);
1818 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, prefix, uIOSize);
1819 }
1820 }
1821 else
1822 {
1823 /* normal in/out */
1824 Assert(!IoExitInfo.n.u1REP);
1825
1826 if (IoExitInfo.n.u1Type == 0)
1827 {
1828 Log2(("IOMIOPortWrite %VGv %x %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
1829 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOWrite);
1830 rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
1831 }
1832 else
1833 {
1834 uint32_t u32Val = 0;
1835
1836 STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIORead);
1837 rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
1838 if (IOM_SUCCESS(rc))
1839 {
1840 /* Write back to the EAX register. */
1841 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
1842 Log2(("IOMIOPortRead %VGv %x %x size=%d\n", pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
1843 }
1844 }
1845 }
1846 /*
1847 * Handled the I/O return codes.
1848 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1849 */
1850 if (IOM_SUCCESS(rc))
1851 {
1852 /* Update EIP and continue execution. */
1853 pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
1854 if (RT_LIKELY(rc == VINF_SUCCESS))
1855 {
1856 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
1857 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
1858 {
1859 STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxIOCheck);
1860 for (unsigned i=0;i<4;i++)
1861 {
1862 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
1863
1864 if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
1865 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
1866 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
1867 {
1868 SVM_EVENT Event;
1869
1870 Assert(CPUMIsGuestDebugStateActive(pVM));
1871
1872 /* Clear all breakpoint status flags and set the one we just hit. */
1873 pCtx->dr[6] &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
1874 pCtx->dr[6] |= (uint64_t)RT_BIT(i);
1875
1876 /* Note: AMD64 Architecture Programmer's Manual 13.1:
1877 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
1878 * the contents have been read.
1879 */
1880 pVMCB->guest.u64DR6 = pCtx->dr[6];
1881
1882 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
1883 pCtx->dr[7] &= ~X86_DR7_GD;
1884
1885 /* Paranoia. */
1886 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1887 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1888 pCtx->dr[7] |= 0x400; /* must be one */
1889
1890 pVMCB->guest.u64DR7 = pCtx->dr[7];
1891
1892 /* Inject the exception. */
1893 Log(("Inject IO debug trap at %VGv\n", pCtx->rip));
1894
1895 Event.au64[0] = 0;
1896 Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1897 Event.n.u1Valid = 1;
1898 Event.n.u8Vector = X86_XCPT_DB;
1899
1900 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1901
1902 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1903 goto ResumeExecution;
1904 }
1905 }
1906 }
1907
1908 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1909 goto ResumeExecution;
1910 }
1911 Log2(("EM status from IO at %VGv %x size %d: %Vrc\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
1912 break;
1913 }
1914
1915#ifdef VBOX_STRICT
1916 if (rc == VINF_IOM_HC_IOPORT_READ)
1917 Assert(IoExitInfo.n.u1Type != 0);
1918 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
1919 Assert(IoExitInfo.n.u1Type == 0);
1920 else
1921 AssertMsg(VBOX_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Vrc\n", rc));
1922#endif
1923 Log2(("Failed IO at %VGv %x size %d\n", pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1924 break;
1925 }
1926
1927 case SVM_EXIT_HLT:
1928 /** Check if external interrupts are pending; if so, don't switch back. */
1929 pCtx->rip++; /* skip hlt */
1930 if ( pCtx->eflags.Bits.u1IF
1931 && VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC|VM_FF_INTERRUPT_PIC)))
1932 goto ResumeExecution;
1933
1934 rc = VINF_EM_HALT;
1935 break;
1936
1937 case SVM_EXIT_RSM:
1938 case SVM_EXIT_INVLPGA:
1939 case SVM_EXIT_VMRUN:
1940 case SVM_EXIT_VMMCALL:
1941 case SVM_EXIT_VMLOAD:
1942 case SVM_EXIT_VMSAVE:
1943 case SVM_EXIT_STGI:
1944 case SVM_EXIT_CLGI:
1945 case SVM_EXIT_SKINIT:
1946 case SVM_EXIT_RDTSCP:
1947 {
1948 /* Unsupported instructions. */
1949 SVM_EVENT Event;
1950
1951 Event.au64[0] = 0;
1952 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1953 Event.n.u1Valid = 1;
1954 Event.n.u8Vector = X86_XCPT_UD;
1955
1956 Log(("Forced #UD trap at %VGv\n", pCtx->rip));
1957 SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1958
1959 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1960 goto ResumeExecution;
1961 }
1962
1963 /* Emulate in ring 3. */
1964 case SVM_EXIT_MSR:
1965 {
1966 uint32_t cbSize;
1967
1968 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
1969 Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
1970 rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1971 if (rc == VINF_SUCCESS)
1972 {
1973 /* EIP has been updated already. */
1974
1975 /* Only resume if successful. */
1976 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1977 goto ResumeExecution;
1978 }
1979 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Vrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
1980 break;
1981 }
1982
1983 case SVM_EXIT_MONITOR:
1984 case SVM_EXIT_RDPMC:
1985 case SVM_EXIT_PAUSE:
1986 case SVM_EXIT_MWAIT_UNCOND:
1987 case SVM_EXIT_MWAIT_ARMED:
1988 case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
1989 rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
1990 break;
1991
1992 case SVM_EXIT_SHUTDOWN:
1993 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
1994 break;
1995
1996 case SVM_EXIT_IDTR_READ:
1997 case SVM_EXIT_GDTR_READ:
1998 case SVM_EXIT_LDTR_READ:
1999 case SVM_EXIT_TR_READ:
2000 case SVM_EXIT_IDTR_WRITE:
2001 case SVM_EXIT_GDTR_WRITE:
2002 case SVM_EXIT_LDTR_WRITE:
2003 case SVM_EXIT_TR_WRITE:
2004 case SVM_EXIT_CR0_SEL_WRITE:
2005 default:
2006 /* Unexpected exit codes. */
2007 rc = VERR_EM_INTERNAL_ERROR;
2008 AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2009 break;
2010 }
2011
2012end:
2013
2014 /* Signal changes for the recompiler. */
2015 CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
2016
2017 /* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
2018 if (exitCode == SVM_EXIT_INTR)
2019 {
2020 STAM_COUNTER_INC(&pVM->hwaccm.s.StatPendingHostIrq);
2021 /* On the next entry we'll only sync the host context. */
2022 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
2023 }
2024 else
2025 {
2026 /* On the next entry we'll sync everything. */
2027 /** @todo we can do better than this */
2028 /* Not in the VINF_PGM_CHANGE_MODE though! */
2029 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
2030 }
2031
2032 /* translate into a less severe return code */
2033 if (rc == VERR_EM_INTERPRETER)
2034 rc = VINF_EM_RAW_EMULATE_INSTR;
2035
2036 STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2037 return rc;
2038}
2039
2040/**
2041 * Enters the AMD-V session
2042 *
2043 * @returns VBox status code.
2044 * @param pVM The VM to operate on.
2045 * @param pCpu CPU info struct
2046 */
2047VMMR0DECL(int) SVMR0Enter(PVM pVM, PHWACCM_CPUINFO pCpu)
2048{
2049 Assert(pVM->hwaccm.s.svm.fSupported);
2050
2051 LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVM->hwaccm.s.idLastCpu, pVM->hwaccm.s.uCurrentASID));
2052 pVM->hwaccm.s.svm.fResumeVM = false;
2053
2054 /* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2055 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_LDTR;
2056
2057 return VINF_SUCCESS;
2058}
2059
2060
2061/**
2062 * Leaves the AMD-V session
2063 *
2064 * @returns VBox status code.
2065 * @param pVM The VM to operate on.
2066 * @param pCtx CPU context
2067 */
2068VMMR0DECL(int) SVMR0Leave(PVM pVM, PCPUMCTX pCtx)
2069{
2070 SVM_VMCB *pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
2071
2072 Assert(pVM->hwaccm.s.svm.fSupported);
2073
2074 /* Save the guest debug state if necessary. */
2075 if (CPUMIsGuestDebugStateActive(pVM))
2076 {
2077 CPUMR0SaveGuestDebugState(pVM, pCtx, false /* skip DR6 */);
2078
2079 /* Intercept all DRx reads and writes again. Changed later on. */
2080 pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2081 pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2082
2083 /* Resync the debug registers the next time. */
2084 pVM->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
2085 }
2086 else
2087 Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2088
2089 return VINF_SUCCESS;
2090}
2091
2092
2093static int svmR0InterpretInvlPg(PVM pVM, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2094{
2095 OP_PARAMVAL param1;
2096 RTGCPTR addr;
2097
2098 int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
2099 if(VBOX_FAILURE(rc))
2100 return VERR_EM_INTERPRETER;
2101
2102 switch(param1.type)
2103 {
2104 case PARMTYPE_IMMEDIATE:
2105 case PARMTYPE_ADDRESS:
2106 if(!(param1.flags & (PARAM_VAL32|PARAM_VAL64)))
2107 return VERR_EM_INTERPRETER;
2108 addr = param1.val.val64;
2109 break;
2110
2111 default:
2112 return VERR_EM_INTERPRETER;
2113 }
2114
2115 /** @todo is addr always a flat linear address or ds based
2116 * (in absence of segment override prefixes)????
2117 */
2118 rc = PGMInvalidatePage(pVM, addr);
2119 if (VBOX_SUCCESS(rc))
2120 {
2121 /* Manually invalidate the page for the VM's TLB. */
2122 Log(("SVMInvlpgA %VGv ASID=%d\n", addr, uASID));
2123 SVMInvlpgA(addr, uASID);
2124 return VINF_SUCCESS;
2125 }
2126 Assert(rc == VERR_REM_FLUSHED_PAGES_OVERFLOW);
2127 return rc;
2128}
2129
2130/**
2131 * Interprets INVLPG
2132 *
2133 * @returns VBox status code.
2134 * @retval VINF_* Scheduling instructions.
2135 * @retval VERR_EM_INTERPRETER Something we can't cope with.
2136 * @retval VERR_* Fatal errors.
2137 *
2138 * @param pVM The VM handle.
2139 * @param pRegFrame The register frame.
2140 * @param ASID Tagged TLB id for the guest
2141 *
2142 * Updates the EIP if an instruction was executed successfully.
2143 */
2144static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2145{
2146 /*
2147 * Only allow 32 & 64 bits code.
2148 */
2149 DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2150 if (enmMode != CPUMODE_16BIT)
2151 {
2152 RTGCPTR pbCode;
2153 int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2154 if (VBOX_SUCCESS(rc))
2155 {
2156 uint32_t cbOp;
2157 DISCPUSTATE Cpu;
2158
2159 Cpu.mode = enmMode;
2160 rc = EMInterpretDisasOneEx(pVM, pbCode, pRegFrame, &Cpu, &cbOp);
2161 Assert(VBOX_FAILURE(rc) || Cpu.pCurInstr->opcode == OP_INVLPG);
2162 if (VBOX_SUCCESS(rc) && Cpu.pCurInstr->opcode == OP_INVLPG)
2163 {
2164 Assert(cbOp == Cpu.opsize);
2165 rc = svmR0InterpretInvlPg(pVM, &Cpu, pRegFrame, uASID);
2166 if (VBOX_SUCCESS(rc))
2167 {
2168 pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2169 }
2170 return rc;
2171 }
2172 }
2173 }
2174 return VERR_EM_INTERPRETER;
2175}
2176
2177
2178/**
2179 * Invalidates a guest page
2180 *
2181 * @returns VBox status code.
2182 * @param pVM The VM to operate on.
2183 * @param GCVirt Page to invalidate
2184 */
2185VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, RTGCPTR GCVirt)
2186{
2187 bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB | pVM->hwaccm.s.fForceTLBFlush;
2188
2189 /* Skip it if a TLB flush is already pending. */
2190 if (!fFlushPending)
2191 {
2192 SVM_VMCB *pVMCB;
2193
2194 Log2(("SVMR0InvalidatePage %VGv\n", GCVirt));
2195 AssertReturn(pVM, VERR_INVALID_PARAMETER);
2196 Assert(pVM->hwaccm.s.svm.fSupported);
2197
2198 pVMCB = (SVM_VMCB *)pVM->hwaccm.s.svm.pVMCB;
2199 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2200
2201 STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushPageManual);
2202 SVMInvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2203 }
2204 return VINF_SUCCESS;
2205}
2206
2207
2208/**
2209 * Invalidates a guest page by physical address
2210 *
2211 * @returns VBox status code.
2212 * @param pVM The VM to operate on.
2213 * @param GCPhys Page to invalidate
2214 */
2215VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, RTGCPHYS GCPhys)
2216{
2217 Assert(pVM->hwaccm.s.fNestedPaging);
2218 /* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
2219 pVM->hwaccm.s.fForceTLBFlush = true;
2220 STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBInvlpga);
2221 return VINF_SUCCESS;
2222}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette