VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 41335

Last change on this file since 41335 was 41335, checked in by vboxsync, 13 years ago

VMM/VMMR0/HWSVMR0: style fixes, cleanup, wrap to 130 columns.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 122.0 KB
Line 
1/* $Id: HWSVMR0.cpp 41335 2012-05-16 12:36:18Z vboxsync $ */
2/** @file
3 * HM SVM (AMD-V) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2012 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HWACCM
22#include <VBox/vmm/hwaccm.h>
23#include <VBox/vmm/pgm.h>
24#include <VBox/vmm/selm.h>
25#include <VBox/vmm/iom.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/tm.h>
29#include <VBox/vmm/pdmapi.h>
30#include "HWACCMInternal.h"
31#include <VBox/vmm/vm.h>
32#include <VBox/vmm/hwacc_svm.h>
33#include <VBox/err.h>
34#include <VBox/log.h>
35#include <VBox/dis.h>
36#include <VBox/disopcode.h>
37#include <iprt/param.h>
38#include <iprt/assert.h>
39#include <iprt/asm.h>
40#include <iprt/asm-amd64-x86.h>
41#include <iprt/cpuset.h>
42#include <iprt/mp.h>
43#include <iprt/time.h>
44#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
45# include <iprt/thread.h>
46#endif
47#include <iprt/x86.h>
48#include "HWSVMR0.h"
49
50#include "dtrace/VBoxVMM.h"
51
52
53/*******************************************************************************
54* Internal Functions *
55*******************************************************************************/
56static int hmR0SvmInterpretInvpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID);
57static int hmR0SvmEmulateTprVMMCall(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
58static void hmR0SvmSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
59
60
61/*******************************************************************************
62* Global Variables *
63*******************************************************************************/
64
65/**
66 * Sets up and activates AMD-V on the current CPU.
67 *
68 * @returns VBox status code.
69 * @param pCpu Pointer to the CPU info struct.
70 * @param pVM Pointer to the VM (can be NULL after a resume!).
71 * @param pvCpuPage Pointer to the global CPU page.
72 * @param HCPhysCpuPage Physical address of the global CPU page.
73 */
74VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
75{
76 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
77 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
78
79 /*
80 * We must turn on AMD-V and setup the host state physical address, as those MSRs are per cpu/core.
81 */
82 uint64_t fEfer = ASMRdMsr(MSR_K6_EFER);
83 if (fEfer & MSR_K6_EFER_SVME)
84 {
85 /*
86 * If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V.
87 */
88 if ( pVM
89 && pVM->hwaccm.s.svm.fIgnoreInUseError)
90 {
91 pCpu->fIgnoreAMDVInUseError = true;
92 }
93
94 if (!pCpu->fIgnoreAMDVInUseError)
95 return VERR_SVM_IN_USE;
96 }
97
98 /* Turn on AMD-V in the EFER MSR. */
99 ASMWrMsr(MSR_K6_EFER, fEfer | MSR_K6_EFER_SVME);
100
101 /* Write the physical page address where the CPU will store the host state while executing the VM. */
102 ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
103
104 /*
105 * Theoretically, other hypervisors may have used ASIDs, ideally we should flush all non-zero ASIDs
106 * when enabling SVM. AMD doesn't have an SVM instruction to flush all ASIDs (flushing is done
107 * upon VMRUN). Therefore, just set the fFlushASIDBeforeUse flag which instructs hmR0SvmSetupTLB()
108 * to flush the TLB with before using a new ASID.
109 */
110 pCpu->fFlushASIDBeforeUse = true;
111 return VINF_SUCCESS;
112}
113
114
115/**
116 * Deactivates AMD-V on the current CPU.
117 *
118 * @returns VBox status code.
119 * @param pCpu Pointer to the CPU info struct.
120 * @param pvCpuPage Pointer to the global CPU page.
121 * @param HCPhysCpuPage Physical address of the global CPU page.
122 */
123VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
124{
125 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
126 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
127 NOREF(pCpu);
128
129 /* Turn off AMD-V in the EFER MSR. */
130 uint64_t fEfer = ASMRdMsr(MSR_K6_EFER);
131 ASMWrMsr(MSR_K6_EFER, fEfer & ~MSR_K6_EFER_SVME);
132
133 /* Invalidate host state physical address. */
134 ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
135
136 return VINF_SUCCESS;
137}
138
139
140/**
141 * Does Ring-0 per VM AMD-V init.
142 *
143 * @returns VBox status code.
144 * @param pVM Pointer to the VM.
145 */
146VMMR0DECL(int) SVMR0InitVM(PVM pVM)
147{
148 int rc;
149
150 pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
151
152 /* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
153 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
154 if (RT_FAILURE(rc))
155 return rc;
156
157 pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
158 pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
159 /* Set all bits to intercept all IO accesses. */
160 ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
161
162 /*
163 * Erratum 170 which requires a forced TLB flush for each world switch:
164 * See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
165 *
166 * All BH-G1/2 and DH-G1/2 models include a fix:
167 * Athlon X2: 0x6b 1/2
168 * 0x68 1/2
169 * Athlon 64: 0x7f 1
170 * 0x6f 2
171 * Sempron: 0x7f 1/2
172 * 0x6f 2
173 * 0x6c 2
174 * 0x7c 2
175 * Turion 64: 0x68 2
176 */
177 uint32_t u32Dummy;
178 uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
179 ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
180 u32BaseFamily = (u32Version >> 8) & 0xf;
181 u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
182 u32Model = ((u32Version >> 4) & 0xf);
183 u32Model = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
184 u32Stepping = u32Version & 0xf;
185 if ( u32Family == 0xf
186 && !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) && u32Stepping >= 1)
187 && !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) && u32Stepping >= 2))
188 {
189 Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
190 pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
191 }
192
193 /* Allocate VMCBs for all guest CPUs. */
194 for (VMCPUID i = 0; i < pVM->cCpus; i++)
195 {
196 PVMCPU pVCpu = &pVM->aCpus[i];
197
198 pVCpu->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
199 pVCpu->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
200 pVCpu->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
201
202 /* Allocate one page for the host context */
203 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
204 if (RT_FAILURE(rc))
205 return rc;
206
207 pVCpu->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVCpu->hwaccm.s.svm.pMemObjVMCBHost);
208 pVCpu->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.svm.pMemObjVMCBHost, 0);
209 Assert(pVCpu->hwaccm.s.svm.pVMCBHostPhys < _4G);
210 ASMMemZeroPage(pVCpu->hwaccm.s.svm.pVMCBHost);
211
212 /* Allocate one page for the VM control block (VMCB). */
213 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
214 if (RT_FAILURE(rc))
215 return rc;
216
217 pVCpu->hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVCpu->hwaccm.s.svm.pMemObjVMCB);
218 pVCpu->hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.svm.pMemObjVMCB, 0);
219 Assert(pVCpu->hwaccm.s.svm.pVMCBPhys < _4G);
220 ASMMemZeroPage(pVCpu->hwaccm.s.svm.pVMCB);
221
222 /* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
223 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
224 if (RT_FAILURE(rc))
225 return rc;
226
227 pVCpu->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVCpu->hwaccm.s.svm.pMemObjMSRBitmap);
228 pVCpu->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.svm.pMemObjMSRBitmap, 0);
229 /* Set all bits to intercept all MSR accesses. */
230 ASMMemFill32(pVCpu->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE * 2, 0xffffffff);
231 }
232
233 return VINF_SUCCESS;
234}
235
236
237/**
238 * Does Ring-0 per VM AMD-V termination.
239 *
240 * @returns VBox status code.
241 * @param pVM Pointer to the VM.
242 */
243VMMR0DECL(int) SVMR0TermVM(PVM pVM)
244{
245 for (VMCPUID i = 0; i < pVM->cCpus; i++)
246 {
247 PVMCPU pVCpu = &pVM->aCpus[i];
248
249 if (pVCpu->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
250 {
251 RTR0MemObjFree(pVCpu->hwaccm.s.svm.pMemObjVMCBHost, false);
252 pVCpu->hwaccm.s.svm.pVMCBHost = 0;
253 pVCpu->hwaccm.s.svm.pVMCBHostPhys = 0;
254 pVCpu->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
255 }
256
257 if (pVCpu->hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
258 {
259 RTR0MemObjFree(pVCpu->hwaccm.s.svm.pMemObjVMCB, false);
260 pVCpu->hwaccm.s.svm.pVMCB = 0;
261 pVCpu->hwaccm.s.svm.pVMCBPhys = 0;
262 pVCpu->hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
263 }
264 if (pVCpu->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
265 {
266 RTR0MemObjFree(pVCpu->hwaccm.s.svm.pMemObjMSRBitmap, false);
267 pVCpu->hwaccm.s.svm.pMSRBitmap = 0;
268 pVCpu->hwaccm.s.svm.pMSRBitmapPhys = 0;
269 pVCpu->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
270 }
271 }
272 if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
273 {
274 RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
275 pVM->hwaccm.s.svm.pIOBitmap = 0;
276 pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
277 pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
278 }
279 return VINF_SUCCESS;
280}
281
282
283/**
284 * Sets up AMD-V for the specified VM.
285 *
286 * @returns VBox status code.
287 * @param pVM Pointer to the VM.
288 */
289VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
290{
291 int rc = VINF_SUCCESS;
292
293 AssertReturn(pVM, VERR_INVALID_PARAMETER);
294 Assert(pVM->hwaccm.s.svm.fSupported);
295
296 for (VMCPUID i = 0; i < pVM->cCpus; i++)
297 {
298 PVMCPU pVCpu = &pVM->aCpus[i];
299 SVM_VMCB *pVMCB = (SVM_VMCB *)pVM->aCpus[i].hwaccm.s.svm.pVMCB;
300
301 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
302
303 /*
304 * Program the control fields. Most of them never have to be changed again.
305 * CR0/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's.
306 * Note: CR0 & CR4 can be safely read when guest and shadow copies are identical.
307 */
308 pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) | RT_BIT(4);
309
310 /* CR0/4 writes must be intercepted for obvious reasons. */
311 pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) | RT_BIT(4);
312
313 /* Intercept all DRx reads and writes by default. Changed later on. */
314 pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
315 pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
316
317 /* Intercept traps; only #NM is always intercepted. */
318 pVMCB->ctrl.u32InterceptException = RT_BIT(X86_XCPT_NM);
319#ifdef VBOX_ALWAYS_TRAP_PF
320 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_PF);
321#endif
322#ifdef VBOX_STRICT
323 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_BP)
324 | RT_BIT(X86_XCPT_DB)
325 | RT_BIT(X86_XCPT_DE)
326 | RT_BIT(X86_XCPT_UD)
327 | RT_BIT(X86_XCPT_NP)
328 | RT_BIT(X86_XCPT_SS)
329 | RT_BIT(X86_XCPT_GP)
330 | RT_BIT(X86_XCPT_MF)
331 ;
332#endif
333
334 /* Set up instruction and miscellaneous intercepts. */
335 pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
336 | SVM_CTRL1_INTERCEPT_VINTR
337 | SVM_CTRL1_INTERCEPT_NMI
338 | SVM_CTRL1_INTERCEPT_SMI
339 | SVM_CTRL1_INTERCEPT_INIT
340 | SVM_CTRL1_INTERCEPT_RDPMC
341 | SVM_CTRL1_INTERCEPT_CPUID
342 | SVM_CTRL1_INTERCEPT_RSM
343 | SVM_CTRL1_INTERCEPT_HLT
344 | SVM_CTRL1_INTERCEPT_INOUT_BITMAP
345 | SVM_CTRL1_INTERCEPT_MSR_SHADOW
346 | SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
347 | SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
348 | SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
349 ;
350 pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
351 | SVM_CTRL2_INTERCEPT_VMMCALL
352 | SVM_CTRL2_INTERCEPT_VMLOAD
353 | SVM_CTRL2_INTERCEPT_VMSAVE
354 | SVM_CTRL2_INTERCEPT_STGI
355 | SVM_CTRL2_INTERCEPT_CLGI
356 | SVM_CTRL2_INTERCEPT_SKINIT
357 | SVM_CTRL2_INTERCEPT_WBINVD
358 | SVM_CTRL2_INTERCEPT_MONITOR
359 | SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the
360 guest (host thinks the cpu load is high) */
361
362 Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
363 Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
364 Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
365
366 /* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
367 pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
368
369 /* Ignore the priority in the TPR; just deliver it when we tell it to. */
370 pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
371
372 /* Set IO and MSR bitmap addresses. */
373 pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
374 pVMCB->ctrl.u64MSRPMPhysAddr = pVCpu->hwaccm.s.svm.pMSRBitmapPhys;
375
376 /* No LBR virtualization. */
377 pVMCB->ctrl.u64LBRVirt = 0;
378
379 /* The ASID must start at 1; the host uses 0. */
380 pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
381
382 /*
383 * Setup the PAT MSR (nested paging only)
384 * The default value should be 0x0007040600070406ULL, but we want to treat all guest memory as WB,
385 * so choose type 6 for all PAT slots.
386 */
387 pVMCB->guest.u64GPAT = 0x0006060606060606ULL;
388
389 /* If nested paging is not in use, additional intercepts have to be set up. */
390 if (!pVM->hwaccm.s.fNestedPaging)
391 {
392 /* CR3 reads/writes must be intercepted; our shadow values are different from guest's. */
393 pVMCB->ctrl.u16InterceptRdCRx |= RT_BIT(3);
394 pVMCB->ctrl.u16InterceptWrCRx |= RT_BIT(3);
395
396 /*
397 * We must also intercept:
398 * - INVLPG (must go through shadow paging)
399 * - task switches (may change CR3/EFLAGS/LDT)
400 */
401 pVMCB->ctrl.u32InterceptCtrl1 |= SVM_CTRL1_INTERCEPT_INVLPG
402 | SVM_CTRL1_INTERCEPT_TASK_SWITCH;
403
404 /* Page faults must be intercepted to implement shadow paging. */
405 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_PF);
406 }
407
408 /*
409 * The following MSRs are saved automatically by vmload/vmsave, so we allow the guest
410 * to modify them directly.
411 */
412 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
413 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_CSTAR, true, true);
414 hmR0SvmSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
415 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
416 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
417 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
418 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
419 hmR0SvmSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
420 hmR0SvmSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
421 hmR0SvmSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
422 }
423
424 return rc;
425}
426
427
428/**
429 * Sets the permission bits for the specified MSR.
430 *
431 * @param pVCpu Pointer to the VMCPU.
432 * @param ulMSR MSR value.
433 * @param fRead Whether reading is allowed.
434 * @param fWrite Whether writing is allowed.
435 */
436static void hmR0SvmSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
437{
438 unsigned ulBit;
439 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.svm.pMSRBitmap;
440
441 if (ulMSR <= 0x00001FFF)
442 {
443 /* Pentium-compatible MSRs */
444 ulBit = ulMSR * 2;
445 }
446 else if ( ulMSR >= 0xC0000000
447 && ulMSR <= 0xC0001FFF)
448 {
449 /* AMD Sixth Generation x86 Processor MSRs and SYSCALL */
450 ulBit = (ulMSR - 0xC0000000) * 2;
451 pMSRBitmap += 0x800;
452 }
453 else if ( ulMSR >= 0xC0010000
454 && ulMSR <= 0xC0011FFF)
455 {
456 /* AMD Seventh and Eighth Generation Processor MSRs */
457 ulBit = (ulMSR - 0xC0001000) * 2;
458 pMSRBitmap += 0x1000;
459 }
460 else
461 {
462 AssertFailed();
463 return;
464 }
465 Assert(ulBit < 16 * 1024 - 1);
466 if (fRead)
467 ASMBitClear(pMSRBitmap, ulBit);
468 else
469 ASMBitSet(pMSRBitmap, ulBit);
470
471 if (fWrite)
472 ASMBitClear(pMSRBitmap, ulBit + 1);
473 else
474 ASMBitSet(pMSRBitmap, ulBit + 1);
475}
476
477
478/**
479 * Injects an event (trap or external interrupt).
480 *
481 * @param pVCpu Pointer to the VMCPU.
482 * @param pVMCB Pointer to the VMCB.
483 * @param pCtx Pointer to the guest CPU context.
484 * @param pIntInfo Pointer to the SVM interrupt info.
485 */
486DECLINLINE(void) hmR0SvmInjectEvent(PVMCPU pVCpu, SVM_VMCB *pVMCB, CPUMCTX *pCtx, SVM_EVENT *pEvent)
487{
488#ifdef VBOX_WITH_STATISTICS
489 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
490#endif
491
492#ifdef VBOX_STRICT
493 if (pEvent->n.u8Vector == 0xE)
494 {
495 Log(("SVM: Inject int %d at %RGv error code=%02x CR2=%RGv intInfo=%08x\n", pEvent->n.u8Vector,
496 (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode, (RTGCPTR)pCtx->cr2, pEvent->au64[0]));
497 }
498 else if (pEvent->n.u8Vector < 0x20)
499 Log(("SVM: Inject int %d at %RGv error code=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode));
500 else
501 {
502 Log(("INJ-EI: %x at %RGv\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip));
503 Assert(!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
504 Assert(pCtx->eflags.u32 & X86_EFL_IF);
505 }
506#endif
507
508 /* Set event injection state. */
509 pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
510}
511
512
513/**
514 * Checks for pending guest interrupts and injects them.
515 *
516 * @returns VBox status code.
517 * @param pVM Pointer to the VM.
518 * @param pVCpu The VM CPU to operate on.
519 * @param pVMCB Pointer to the VMCB.
520 * @param pCtx Pointer to the guest CPU Context.
521 */
522static int hmR0SvmCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, SVM_VMCB *pVMCB, CPUMCTX *pCtx)
523{
524 int rc;
525 NOREF(pVM);
526
527 /*
528 * Dispatch any pending interrupts (injected before, but a VM-exit occurred prematurely).
529 */
530 if (pVCpu->hwaccm.s.Event.fPending)
531 {
532 SVM_EVENT Event;
533
534 Log(("Reinjecting event %08x %08x at %RGv\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode,
535 (RTGCPTR)pCtx->rip));
536 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
537 Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
538 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
539
540 pVCpu->hwaccm.s.Event.fPending = false;
541 return VINF_SUCCESS;
542 }
543
544 /*
545 * If an active trap is already pending, we must forward it first!
546 */
547 if (!TRPMHasTrap(pVCpu))
548 {
549 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
550 {
551 SVM_EVENT Event;
552
553 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
554 Event.n.u8Vector = X86_XCPT_NMI;
555 Event.n.u1Valid = 1;
556 Event.n.u32ErrorCode = 0;
557 Event.n.u3Type = SVM_EVENT_NMI;
558
559 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
560 return VINF_SUCCESS;
561 }
562
563 /** @todo SMI interrupts. */
564
565 /*
566 * When external interrupts are pending, we should exit the VM when IF is set.
567 */
568 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
569 {
570 if ( !(pCtx->eflags.u32 & X86_EFL_IF)
571 || VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
572 {
573 if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
574 {
575 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
576 LogFlow(("Enable irq window exit!\n"));
577 else
578 {
579 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n",
580 (RTGCPTR)pCtx->rip));
581 }
582
583 /** @todo Use virtual interrupt method to inject a pending IRQ; dispatched as
584 * soon as guest.IF is set. */
585 pVMCB->ctrl.u32InterceptCtrl1 |= SVM_CTRL1_INTERCEPT_VINTR;
586 pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
587 pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
588 }
589 }
590 else
591 {
592 uint8_t u8Interrupt;
593
594 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
595 Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
596 if (RT_SUCCESS(rc))
597 {
598 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
599 AssertRC(rc);
600 }
601 else
602 {
603 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
604 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
605 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
606 /* Just continue */
607 }
608 }
609 }
610 }
611
612#ifdef VBOX_STRICT
613 if (TRPMHasTrap(pVCpu))
614 {
615 uint8_t u8Vector;
616 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
617 AssertRC(rc);
618 }
619#endif
620
621 if ( (pCtx->eflags.u32 & X86_EFL_IF)
622 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
623 && TRPMHasTrap(pVCpu)
624 )
625 {
626 uint8_t u8Vector;
627 TRPMEVENT enmType;
628 SVM_EVENT Event;
629 RTGCUINT u32ErrorCode;
630
631 Event.au64[0] = 0;
632
633 /* If a new event is pending, then dispatch it now. */
634 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &u32ErrorCode, 0);
635 AssertRC(rc);
636 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
637 Assert(enmType != TRPM_SOFTWARE_INT);
638
639 /* Clear the pending trap. */
640 rc = TRPMResetTrap(pVCpu);
641 AssertRC(rc);
642
643 Event.n.u8Vector = u8Vector;
644 Event.n.u1Valid = 1;
645 Event.n.u32ErrorCode = u32ErrorCode;
646
647 if (enmType == TRPM_TRAP)
648 {
649 switch (u8Vector)
650 {
651 case X86_XCPT_DF:
652 case X86_XCPT_TS:
653 case X86_XCPT_NP:
654 case X86_XCPT_SS:
655 case X86_XCPT_GP:
656 case X86_XCPT_PF:
657 case X86_XCPT_AC:
658 /* Valid error codes. */
659 Event.n.u1ErrorCodeValid = 1;
660 break;
661 default:
662 break;
663 }
664 if (u8Vector == X86_XCPT_NMI)
665 Event.n.u3Type = SVM_EVENT_NMI;
666 else
667 Event.n.u3Type = SVM_EVENT_EXCEPTION;
668 }
669 else
670 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
671
672 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
673 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
674 } /* if (interrupts can be dispatched) */
675
676 return VINF_SUCCESS;
677}
678
679
680/**
681 * Save the host state.
682 *
683 * @returns VBox status code.
684 * @param pVM Pointer to the VM.
685 * @param pVCpu The VM CPU to operate on.
686 */
687VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
688{
689 NOREF(pVM);
690 NOREF(pVCpu);
691 /* Nothing to do here. */
692 return VINF_SUCCESS;
693}
694
695
696/**
697 * Loads the guest state.
698 *
699 * NOTE: Don't do anything here that can cause a jump back to ring-3!!!
700 *
701 * @returns VBox status code.
702 * @param pVM Pointer to the VM.
703 * @param pVCpu The VM CPU to operate on.
704 * @param pCtx Pointer to the guest CPU context.
705 */
706VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
707{
708 RTGCUINTPTR val;
709 SVM_VMCB *pVMCB;
710
711 if (pVM == NULL)
712 return VERR_INVALID_PARAMETER;
713
714 /* Setup AMD SVM. */
715 Assert(pVM->hwaccm.s.svm.fSupported);
716
717 pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
718 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
719
720 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
721 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
722 {
723 SVM_WRITE_SELREG(CS, cs);
724 SVM_WRITE_SELREG(SS, ss);
725 SVM_WRITE_SELREG(DS, ds);
726 SVM_WRITE_SELREG(ES, es);
727 SVM_WRITE_SELREG(FS, fs);
728 SVM_WRITE_SELREG(GS, gs);
729 }
730
731 /* Guest CPU context: LDTR. */
732 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
733 {
734 SVM_WRITE_SELREG(LDTR, ldtr);
735 }
736
737 /* Guest CPU context: TR. */
738 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
739 {
740 SVM_WRITE_SELREG(TR, tr);
741 }
742
743 /* Guest CPU context: GDTR. */
744 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
745 {
746 pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
747 pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
748 }
749
750 /* Guest CPU context: IDTR. */
751 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
752 {
753 pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
754 pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
755 }
756
757 /*
758 * Sysenter MSRs (unconditional)
759 */
760 pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
761 pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
762 pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
763
764 /* Control registers */
765 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
766 {
767 val = pCtx->cr0;
768 if (!CPUMIsGuestFPUStateActive(pVCpu))
769 {
770 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
771 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
772 }
773 else
774 {
775 /** @todo check if we support the old style mess correctly. */
776 if (!(val & X86_CR0_NE))
777 {
778 Log(("Forcing X86_CR0_NE!!!\n"));
779
780 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
781 if (!pVCpu->hwaccm.s.fFPUOldStyleOverride)
782 {
783 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_MF);
784 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
785 }
786 }
787 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
788 }
789 /* Always enable caching. */
790 val &= ~(X86_CR0_CD|X86_CR0_NW);
791
792 /*
793 * Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level.
794 * Note: In nested paging mode, the guest is allowed to run with paging disabled; the guest-physical to host-physical
795 * translation will remain active.
796 */
797 if (!pVM->hwaccm.s.fNestedPaging)
798 {
799 val |= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
800 val |= X86_CR0_WP; /* Must set this as we rely on protecting various pages and supervisor writes must be caught. */
801 }
802 pVMCB->guest.u64CR0 = val;
803 }
804 /* CR2 as well */
805 pVMCB->guest.u64CR2 = pCtx->cr2;
806
807 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
808 {
809 /* Save our shadow CR3 register. */
810 if (pVM->hwaccm.s.fNestedPaging)
811 {
812 PGMMODE enmShwPagingMode;
813
814#if HC_ARCH_BITS == 32
815 if (CPUMIsGuestInLongModeEx(pCtx))
816 enmShwPagingMode = PGMMODE_AMD64_NX;
817 else
818#endif
819 enmShwPagingMode = PGMGetHostMode(pVM);
820
821 pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
822 Assert(pVMCB->ctrl.u64NestedPagingCR3);
823 pVMCB->guest.u64CR3 = pCtx->cr3;
824 }
825 else
826 {
827 pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
828 Assert(pVMCB->guest.u64CR3 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
829 }
830 }
831
832 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
833 {
834 val = pCtx->cr4;
835 if (!pVM->hwaccm.s.fNestedPaging)
836 {
837 switch(pVCpu->hwaccm.s.enmShadowMode)
838 {
839 case PGMMODE_REAL:
840 case PGMMODE_PROTECTED: /* Protected mode, no paging. */
841 AssertFailed();
842 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
843
844 case PGMMODE_32_BIT: /* 32-bit paging. */
845 val &= ~X86_CR4_PAE;
846 break;
847
848 case PGMMODE_PAE: /* PAE paging. */
849 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
850 /** Must use PAE paging as we could use physical memory > 4 GB */
851 val |= X86_CR4_PAE;
852 break;
853
854 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
855 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
856#ifdef VBOX_ENABLE_64_BITS_GUESTS
857 break;
858#else
859 AssertFailed();
860 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
861#endif
862
863 default: /* shut up gcc */
864 AssertFailed();
865 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
866 }
867 }
868 pVMCB->guest.u64CR4 = val;
869 }
870
871 /* Debug registers. */
872 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
873 {
874 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
875 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
876
877 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
878 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
879 pCtx->dr[7] |= 0x400; /* must be one */
880
881 pVMCB->guest.u64DR7 = pCtx->dr[7];
882 pVMCB->guest.u64DR6 = pCtx->dr[6];
883
884#ifdef DEBUG
885 /* Sync the hypervisor debug state now if any breakpoint is armed. */
886 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
887 && !CPUMIsHyperDebugStateActive(pVCpu)
888 && !DBGFIsStepping(pVCpu))
889 {
890 /* Save the host and load the hypervisor debug state. */
891 int rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
892 AssertRC(rc);
893
894 /* DRx intercepts remain enabled. */
895
896 /* Override dr6 & dr7 with the hypervisor values. */
897 pVMCB->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
898 pVMCB->guest.u64DR6 = CPUMGetHyperDR6(pVCpu);
899 }
900 else
901#endif
902 /* Sync the debug state now if any breakpoint is armed. */
903 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
904 && !CPUMIsGuestDebugStateActive(pVCpu)
905 && !DBGFIsStepping(pVCpu))
906 {
907 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
908
909 /* Disable drx move intercepts. */
910 pVMCB->ctrl.u16InterceptRdDRx = 0;
911 pVMCB->ctrl.u16InterceptWrDRx = 0;
912
913 /* Save the host and load the guest debug state. */
914 int rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
915 AssertRC(rc);
916 }
917 }
918
919 /* EIP, ESP and EFLAGS */
920 pVMCB->guest.u64RIP = pCtx->rip;
921 pVMCB->guest.u64RSP = pCtx->rsp;
922 pVMCB->guest.u64RFlags = pCtx->eflags.u32;
923
924 /* Set CPL */
925 pVMCB->guest.u8CPL = pCtx->ssHid.Attr.n.u2Dpl;
926
927 /* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
928 pVMCB->guest.u64RAX = pCtx->rax;
929
930 /* vmrun will fail without MSR_K6_EFER_SVME. */
931 pVMCB->guest.u64EFER = pCtx->msrEFER | MSR_K6_EFER_SVME;
932
933 /* 64 bits guest mode? */
934 if (CPUMIsGuestInLongModeEx(pCtx))
935 {
936#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
937 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
938#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
939 pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
940#else
941# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
942 if (!pVM->hwaccm.s.fAllow64BitGuests)
943 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
944# endif
945 pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun64;
946#endif
947 /* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
948 pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
949 pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
950 }
951 else
952 {
953 /* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
954 pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
955
956 pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun;
957 }
958
959 /* TSC offset. */
960 if (TMCpuTickCanUseRealTSC(pVCpu, &pVMCB->ctrl.u64TSCOffset))
961 {
962 uint64_t u64CurTSC = ASMReadTSC();
963 if (u64CurTSC + pVMCB->ctrl.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
964 {
965 pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
966 pVMCB->ctrl.u32InterceptCtrl2 &= ~SVM_CTRL2_INTERCEPT_RDTSCP;
967 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
968 }
969 else
970 {
971 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
972 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
973 pVMCB->ctrl.u64TSCOffset, u64CurTSC + pVMCB->ctrl.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVMCB->ctrl.u64TSCOffset, TMCpuTickGet(pVCpu)));
974 pVMCB->ctrl.u32InterceptCtrl1 |= SVM_CTRL1_INTERCEPT_RDTSC;
975 pVMCB->ctrl.u32InterceptCtrl2 |= SVM_CTRL2_INTERCEPT_RDTSCP;
976 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
977 }
978 }
979 else
980 {
981 pVMCB->ctrl.u32InterceptCtrl1 |= SVM_CTRL1_INTERCEPT_RDTSC;
982 pVMCB->ctrl.u32InterceptCtrl2 |= SVM_CTRL2_INTERCEPT_RDTSCP;
983 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
984 }
985
986 /* Sync the various MSRs for 64-bit mode. */
987 pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
988 pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64-bit mode syscall rip */
989 pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
990 pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
991 pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* SWAPGS exchange value */
992
993#ifdef DEBUG
994 /* Intercept X86_XCPT_DB if stepping is enabled */
995 if ( DBGFIsStepping(pVCpu)
996 || CPUMIsHyperDebugStateActive(pVCpu))
997 pVMCB->ctrl.u32InterceptException |= RT_BIT(X86_XCPT_DB);
998 else
999 pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
1000#endif
1001
1002 /* Done. */
1003 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1004
1005 return VINF_SUCCESS;
1006}
1007
1008
1009/**
1010 * Setup TLB for ASID.
1011 *
1012 * @param pVM Pointer to the VM.
1013 * @param pVCpu The VM CPU to operate on.
1014 */
1015static void hmR0SvmSetupTLB(PVM pVM, PVMCPU pVCpu)
1016{
1017 PHMGLOBLCPUINFO pCpu;
1018
1019 AssertPtr(pVM);
1020 AssertPtr(pVCpu);
1021
1022 SVM_VMCB *pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
1023 pCpu = HWACCMR0GetCurrentCpu();
1024
1025 /*
1026 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1027 * This can happen both for start & resume due to long jumps back to ring-3.
1028 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
1029 * so we cannot reuse the ASIDs without flushing.
1030 */
1031 bool fNewASID = false;
1032 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1033 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1034 {
1035 pVCpu->hwaccm.s.fForceTLBFlush = true;
1036 fNewASID = true;
1037 }
1038
1039 /*
1040 * Set TLB flush state as checked until we return from the world switch.
1041 */
1042 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
1043
1044 /*
1045 * Check for TLB shootdown flushes.
1046 */
1047 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1048 pVCpu->hwaccm.s.fForceTLBFlush = true;
1049
1050 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1051 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1052
1053 if (RT_UNLIKELY(pVM->hwaccm.s.svm.fAlwaysFlushTLB))
1054 {
1055 /*
1056 * This is the AMD erratum 170. We need to flush the entire TLB for each world switch. Sad.
1057 */
1058 pCpu->uCurrentASID = 1;
1059 pVCpu->hwaccm.s.uCurrentASID = 1;
1060 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1061 }
1062 else if (pVCpu->hwaccm.s.fForceTLBFlush)
1063 {
1064 if (fNewASID)
1065 {
1066 ++pCpu->uCurrentASID;
1067 bool fHitASIDLimit = false;
1068 if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
1069 {
1070 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1071 pCpu->cTLBFlushes++;
1072 fHitASIDLimit = true;
1073
1074 if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1075 {
1076 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1077 pCpu->fFlushASIDBeforeUse = true;
1078 }
1079 else
1080 {
1081 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1082 pCpu->fFlushASIDBeforeUse = false;
1083 }
1084 }
1085
1086 if ( !fHitASIDLimit
1087 && pCpu->fFlushASIDBeforeUse)
1088 {
1089 if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1090 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1091 else
1092 {
1093 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1094 pCpu->fFlushASIDBeforeUse = false;
1095 }
1096 }
1097
1098 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1099 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1100 }
1101 else
1102 {
1103 if (pVM->hwaccm.s.svm.u32Features & AMD_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1104 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1105 else
1106 pVMCB->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1107 }
1108
1109 pVCpu->hwaccm.s.fForceTLBFlush = false;
1110 }
1111 else
1112 {
1113 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1114 * not be executed. See hwaccmQueueInvlPage() where it is commented
1115 * out. Support individual entry flushing someday. */
1116 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1117 {
1118 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1119 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
1120 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
1121 SVMR0InvlpgA(pVCpu->hwaccm.s.TlbShootdown.aPages[i], pVMCB->ctrl.TLBCtrl.n.u32ASID);
1122 }
1123 }
1124
1125 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
1126 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1127
1128 /* Update VMCB with the ASID. */
1129 pVMCB->ctrl.TLBCtrl.n.u32ASID = pVCpu->hwaccm.s.uCurrentASID;
1130
1131 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
1132 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1133 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
1134 ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1135 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
1136 ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1137
1138#ifdef VBOX_WITH_STATISTICS
1139 if (pVMCB->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1140 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1141 else if ( pVMCB->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1142 || pVMCB->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1143 {
1144 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1145 }
1146 else
1147 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1148#endif
1149}
1150
1151
1152/**
1153 * Runs guest code in an AMD-V VM.
1154 *
1155 * @returns VBox status code.
1156 * @param pVM Pointer to the VM.
1157 * @param pVCpu The VM CPU to operate on.
1158 * @param pCtx Pointer to the guest CPU context.
1159 */
1160VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1161{
1162 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1163 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
1164 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
1165
1166 VBOXSTRICTRC rc = VINF_SUCCESS;
1167 int rc2;
1168 uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
1169 SVM_VMCB *pVMCB = NULL;
1170 bool fSyncTPR = false;
1171 unsigned cResume = 0;
1172 uint8_t u8LastTPR = 0; /* Initialized for potentially stupid compilers. */
1173 PHMGLOBLCPUINFO pCpu = 0;
1174 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
1175#ifdef VBOX_STRICT
1176 RTCPUID idCpuCheck;
1177#endif
1178#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
1179 uint64_t u64LastTime = RTTimeMilliTS();
1180#endif
1181
1182 pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
1183 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
1184
1185 /*
1186 * We can jump to this point to resume execution after determining that a VM-exit is innocent.
1187 */
1188ResumeExecution:
1189 if (!STAM_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
1190 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
1191 Assert(!HWACCMR0SuspendPending());
1192
1193 /*
1194 * Safety precaution; looping for too long here can have a very bad effect on the host.
1195 */
1196 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
1197 {
1198 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
1199 rc = VINF_EM_RAW_INTERRUPT;
1200 goto end;
1201 }
1202
1203 /*
1204 * Check for IRQ inhibition due to instruction fusing (sti, mov ss).
1205 */
1206 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1207 {
1208 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
1209 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
1210 {
1211 /*
1212 * Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1213 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1214 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1215 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
1216 */
1217 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1218 /* Irq inhibition is no longer active; clear the corresponding SVM state. */
1219 pVMCB->ctrl.u64IntShadow = 0;
1220 }
1221 }
1222 else
1223 {
1224 /* Irq inhibition is no longer active; clear the corresponding SVM state. */
1225 pVMCB->ctrl.u64IntShadow = 0;
1226 }
1227
1228#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
1229 if (RT_UNLIKELY((cResume & 0xf) == 0))
1230 {
1231 uint64_t u64CurTime = RTTimeMilliTS();
1232
1233 if (RT_UNLIKELY(u64CurTime > u64LastTime))
1234 {
1235 u64LastTime = u64CurTime;
1236 TMTimerPollVoid(pVM, pVCpu);
1237 }
1238 }
1239#endif
1240
1241 /*
1242 * Check for pending actions that force us to go back to ring-3.
1243 */
1244 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
1245 || VMCPU_FF_ISPENDING(pVCpu,
1246 VMCPU_FF_HWACCM_TO_R3_MASK
1247 | VMCPU_FF_PGM_SYNC_CR3
1248 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
1249 | VMCPU_FF_REQUEST))
1250 {
1251 /* Check if a sync operation is pending. */
1252 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
1253 {
1254 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
1255 AssertRC(VBOXSTRICTRC_VAL(rc));
1256 if (rc != VINF_SUCCESS)
1257 {
1258 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
1259 goto end;
1260 }
1261 }
1262
1263#ifdef DEBUG
1264 /* Intercept X86_XCPT_DB if stepping is enabled */
1265 if (!DBGFIsStepping(pVCpu))
1266#endif
1267 {
1268 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
1269 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
1270 {
1271 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
1272 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
1273 goto end;
1274 }
1275 }
1276
1277 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
1278 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
1279 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
1280 {
1281 rc = VINF_EM_PENDING_REQUEST;
1282 goto end;
1283 }
1284
1285 /* Check if a pgm pool flush is in progress. */
1286 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
1287 {
1288 rc = VINF_PGM_POOL_FLUSH_PENDING;
1289 goto end;
1290 }
1291
1292 /* Check if DMA work is pending (2nd+ run). */
1293 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
1294 {
1295 rc = VINF_EM_RAW_TO_R3;
1296 goto end;
1297 }
1298 }
1299
1300#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1301 /*
1302 * Exit to ring-3 preemption/work is pending.
1303 *
1304 * Interrupts are disabled before the call to make sure we don't miss any interrupt
1305 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
1306 * further down, but hmR0SvmCheckPendingInterrupt makes that impossible.)
1307 *
1308 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
1309 * shootdowns rely on this.
1310 */
1311 uOldEFlags = ASMIntDisableFlags();
1312 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
1313 {
1314 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
1315 rc = VINF_EM_RAW_INTERRUPT;
1316 goto end;
1317 }
1318 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1319#endif
1320
1321 /*
1322 * When external interrupts are pending, we should exit the VM when IF is set.
1323 * Note: *After* VM_FF_INHIBIT_INTERRUPTS check!!
1324 */
1325 rc = hmR0SvmCheckPendingInterrupt(pVM, pVCpu, pVMCB, pCtx);
1326 if (RT_FAILURE(rc))
1327 goto end;
1328
1329 /*
1330 * TPR caching using CR8 is only available in 64-bit mode or with 32-bit guests when X86_CPUID_AMD_FEATURE_ECX_CR8L is
1331 * supported.
1332 * Note: we can't do this in LoddGuestState as PDMApicGetTPR can jump back to ring 3 (lock)! (no longer true)
1333 */
1334 /** @todo query and update the TPR only when it could have been changed (mmio access)
1335 */
1336 if (pVM->hwaccm.s.fHasIoApic)
1337 {
1338 /* TPR caching in CR8 */
1339 bool fPending;
1340 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
1341 AssertRC(rc2);
1342
1343 if (pVM->hwaccm.s.fTPRPatchingActive)
1344 {
1345 /* Our patch code uses LSTAR for TPR caching. */
1346 pCtx->msrLSTAR = u8LastTPR;
1347
1348 if (fPending)
1349 {
1350 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
1351 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
1352 }
1353 else
1354 {
1355 /*
1356 * No interrupts are pending, so we don't need to be explicitely notified.
1357 * There are enough world switches for detecting pending interrupts.
1358 */
1359 hmR0SvmSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
1360 }
1361 }
1362 else
1363 {
1364 pVMCB->ctrl.IntCtrl.n.u8VTPR = (u8LastTPR >> 4); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
1365
1366 if (fPending)
1367 {
1368 /* A TPR change could activate a pending interrupt, so catch cr8 writes. */
1369 pVMCB->ctrl.u16InterceptWrCRx |= RT_BIT(8);
1370 }
1371 else
1372 {
1373 /*
1374 * No interrupts are pending, so we don't need to be explicitely notified.
1375 * There are enough world switches for detecting pending interrupts.
1376 */
1377 pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
1378 }
1379 }
1380 fSyncTPR = !fPending;
1381 }
1382
1383 /* All done! Let's start VM execution. */
1384
1385 /* Enable nested paging if necessary (disabled each time after #VMEXIT). */
1386 pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
1387
1388#ifdef LOG_ENABLED
1389 pCpu = HWACCMR0GetCurrentCpu();
1390 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1391 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1392 {
1393 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
1394 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
1395 else
1396 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1397 }
1398 else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
1399 LogFlow(("Manual TLB flush\n"));
1400#endif
1401
1402 /*
1403 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
1404 * (until the actual world switch)
1405 */
1406#ifdef VBOX_STRICT
1407 idCpuCheck = RTMpCpuId();
1408#endif
1409 VMMR0LogFlushDisable(pVCpu);
1410
1411 /*
1412 * Load the guest state; *must* be here as it sets up the shadow CR0 for lazy FPU syncing!
1413 */
1414 rc = SVMR0LoadGuestState(pVM, pVCpu, pCtx);
1415 if (RT_UNLIKELY(rc != VINF_SUCCESS))
1416 {
1417 VMMR0LogFlushEnable(pVCpu);
1418 goto end;
1419 }
1420
1421#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1422 /*
1423 * Disable interrupts to make sure a poke will interrupt execution.
1424 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
1425 */
1426 uOldEFlags = ASMIntDisableFlags();
1427 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
1428#endif
1429 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
1430
1431 /* Setup TLB control and ASID in the VMCB. */
1432 hmR0SvmSetupTLB(pVM, pVCpu);
1433
1434 /* In case we execute a goto ResumeExecution later on. */
1435 pVCpu->hwaccm.s.fResumeVM = true;
1436 pVCpu->hwaccm.s.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
1437
1438 Assert(sizeof(pVCpu->hwaccm.s.svm.pVMCBPhys) == 8);
1439 Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1440 Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
1441 Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVCpu->hwaccm.s.svm.pMSRBitmapPhys);
1442 Assert(pVMCB->ctrl.u64LBRVirt == 0);
1443
1444#ifdef VBOX_STRICT
1445 Assert(idCpuCheck == RTMpCpuId());
1446#endif
1447 TMNotifyStartOfExecution(pVCpu);
1448#ifdef VBOX_WITH_KERNEL_USING_XMM
1449 hwaccmR0SVMRunWrapXMM(pVCpu->hwaccm.s.svm.pVMCBHostPhys, pVCpu->hwaccm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu, pVCpu->hwaccm.s.svm.pfnVMRun);
1450#else
1451 pVCpu->hwaccm.s.svm.pfnVMRun(pVCpu->hwaccm.s.svm.pVMCBHostPhys, pVCpu->hwaccm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu);
1452#endif
1453 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
1454 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
1455 /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
1456 if (!(pVMCB->ctrl.u32InterceptCtrl1 & SVM_CTRL1_INTERCEPT_RDTSC))
1457 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVMCB->ctrl.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
1458 TMNotifyEndOfExecution(pVCpu);
1459 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
1460 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
1461 ASMSetFlags(uOldEFlags);
1462#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
1463 uOldEFlags = ~(RTCCUINTREG)0;
1464#endif
1465
1466 /*
1467 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1468 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1469 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1470 */
1471
1472 /* Reason for the VM exit */
1473 exitCode = pVMCB->ctrl.u64ExitCode;
1474
1475 if (RT_UNLIKELY(exitCode == (uint64_t)SVM_EXIT_INVALID)) /* Invalid guest state. */
1476 {
1477 HWACCMDumpRegs(pVM, pVCpu, pCtx);
1478#ifdef DEBUG
1479 Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1480 Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1481 Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1482 Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1483 Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1484 Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1485 Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1486 Log(("ctrl.u64IOPMPhysAddr %RX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1487 Log(("ctrl.u64MSRPMPhysAddr %RX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1488 Log(("ctrl.u64TSCOffset %RX64\n", pVMCB->ctrl.u64TSCOffset));
1489
1490 Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1491 Log(("ctrl.TLBCtrl.u8TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u8TLBFlush));
1492 Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1493
1494 Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1495 Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1496 Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1497 Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1498 Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1499 Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1500 Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1501 Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1502 Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1503 Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1504
1505 Log(("ctrl.u64IntShadow %RX64\n", pVMCB->ctrl.u64IntShadow));
1506 Log(("ctrl.u64ExitCode %RX64\n", pVMCB->ctrl.u64ExitCode));
1507 Log(("ctrl.u64ExitInfo1 %RX64\n", pVMCB->ctrl.u64ExitInfo1));
1508 Log(("ctrl.u64ExitInfo2 %RX64\n", pVMCB->ctrl.u64ExitInfo2));
1509 Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1510 Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1511 Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1512 Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1513 Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1514 Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1515 Log(("ctrl.NestedPaging %RX64\n", pVMCB->ctrl.NestedPaging.au64));
1516 Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1517 Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1518 Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1519 Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1520 Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1521 Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1522
1523 Log(("ctrl.u64NestedPagingCR3 %RX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1524 Log(("ctrl.u64LBRVirt %RX64\n", pVMCB->ctrl.u64LBRVirt));
1525
1526 Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1527 Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1528 Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1529 Log(("guest.CS.u64Base %RX64\n", pVMCB->guest.CS.u64Base));
1530 Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1531 Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1532 Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1533 Log(("guest.DS.u64Base %RX64\n", pVMCB->guest.DS.u64Base));
1534 Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1535 Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1536 Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1537 Log(("guest.ES.u64Base %RX64\n", pVMCB->guest.ES.u64Base));
1538 Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1539 Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1540 Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1541 Log(("guest.FS.u64Base %RX64\n", pVMCB->guest.FS.u64Base));
1542 Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1543 Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1544 Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1545 Log(("guest.GS.u64Base %RX64\n", pVMCB->guest.GS.u64Base));
1546
1547 Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1548 Log(("guest.GDTR.u64Base %RX64\n", pVMCB->guest.GDTR.u64Base));
1549
1550 Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1551 Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1552 Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1553 Log(("guest.LDTR.u64Base %RX64\n", pVMCB->guest.LDTR.u64Base));
1554
1555 Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1556 Log(("guest.IDTR.u64Base %RX64\n", pVMCB->guest.IDTR.u64Base));
1557
1558 Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1559 Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1560 Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1561 Log(("guest.TR.u64Base %RX64\n", pVMCB->guest.TR.u64Base));
1562
1563 Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1564 Log(("guest.u64CR0 %RX64\n", pVMCB->guest.u64CR0));
1565 Log(("guest.u64CR2 %RX64\n", pVMCB->guest.u64CR2));
1566 Log(("guest.u64CR3 %RX64\n", pVMCB->guest.u64CR3));
1567 Log(("guest.u64CR4 %RX64\n", pVMCB->guest.u64CR4));
1568 Log(("guest.u64DR6 %RX64\n", pVMCB->guest.u64DR6));
1569 Log(("guest.u64DR7 %RX64\n", pVMCB->guest.u64DR7));
1570
1571 Log(("guest.u64RIP %RX64\n", pVMCB->guest.u64RIP));
1572 Log(("guest.u64RSP %RX64\n", pVMCB->guest.u64RSP));
1573 Log(("guest.u64RAX %RX64\n", pVMCB->guest.u64RAX));
1574 Log(("guest.u64RFlags %RX64\n", pVMCB->guest.u64RFlags));
1575
1576 Log(("guest.u64SysEnterCS %RX64\n", pVMCB->guest.u64SysEnterCS));
1577 Log(("guest.u64SysEnterEIP %RX64\n", pVMCB->guest.u64SysEnterEIP));
1578 Log(("guest.u64SysEnterESP %RX64\n", pVMCB->guest.u64SysEnterESP));
1579
1580 Log(("guest.u64EFER %RX64\n", pVMCB->guest.u64EFER));
1581 Log(("guest.u64STAR %RX64\n", pVMCB->guest.u64STAR));
1582 Log(("guest.u64LSTAR %RX64\n", pVMCB->guest.u64LSTAR));
1583 Log(("guest.u64CSTAR %RX64\n", pVMCB->guest.u64CSTAR));
1584 Log(("guest.u64SFMASK %RX64\n", pVMCB->guest.u64SFMASK));
1585 Log(("guest.u64KernelGSBase %RX64\n", pVMCB->guest.u64KernelGSBase));
1586 Log(("guest.u64GPAT %RX64\n", pVMCB->guest.u64GPAT));
1587 Log(("guest.u64DBGCTL %RX64\n", pVMCB->guest.u64DBGCTL));
1588 Log(("guest.u64BR_FROM %RX64\n", pVMCB->guest.u64BR_FROM));
1589 Log(("guest.u64BR_TO %RX64\n", pVMCB->guest.u64BR_TO));
1590 Log(("guest.u64LASTEXCPFROM %RX64\n", pVMCB->guest.u64LASTEXCPFROM));
1591 Log(("guest.u64LASTEXCPTO %RX64\n", pVMCB->guest.u64LASTEXCPTO));
1592
1593#endif
1594 rc = VERR_SVM_UNABLE_TO_START_VM;
1595 VMMR0LogFlushEnable(pVCpu);
1596 goto end;
1597 }
1598
1599 /* Let's first sync back EIP, ESP, and EFLAGS. */
1600 pCtx->rip = pVMCB->guest.u64RIP;
1601 pCtx->rsp = pVMCB->guest.u64RSP;
1602 pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1603 /* eax is saved/restore across the vmrun instruction */
1604 pCtx->rax = pVMCB->guest.u64RAX;
1605
1606 /*
1607 * Save all the MSRs that can be changed by the guest without causing a world switch.
1608 * FS & GS base are saved with SVM_READ_SELREG.
1609 */
1610 pCtx->msrSTAR = pVMCB->guest.u64STAR; /* legacy syscall eip, cs & ss */
1611 pCtx->msrLSTAR = pVMCB->guest.u64LSTAR; /* 64-bit mode syscall rip */
1612 pCtx->msrCSTAR = pVMCB->guest.u64CSTAR; /* compatibility mode syscall rip */
1613 pCtx->msrSFMASK = pVMCB->guest.u64SFMASK; /* syscall flag mask */
1614 pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1615 pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1616 pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1617 pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1618
1619 /* Can be updated behind our back in the nested paging case. */
1620 pCtx->cr2 = pVMCB->guest.u64CR2;
1621
1622 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1623 SVM_READ_SELREG(SS, ss);
1624 SVM_READ_SELREG(CS, cs);
1625 SVM_READ_SELREG(DS, ds);
1626 SVM_READ_SELREG(ES, es);
1627 SVM_READ_SELREG(FS, fs);
1628 SVM_READ_SELREG(GS, gs);
1629
1630 /*
1631 * Correct the hidden CS granularity flag. Haven't seen it being wrong in any other
1632 * register (yet).
1633 */
1634 if ( !pCtx->csHid.Attr.n.u1Granularity
1635 && pCtx->csHid.Attr.n.u1Present
1636 && pCtx->csHid.u32Limit > UINT32_C(0xfffff))
1637 {
1638 Assert((pCtx->csHid.u32Limit & 0xfff) == 0xfff);
1639 pCtx->csHid.Attr.n.u1Granularity = 1;
1640 }
1641#define SVM_ASSERT_SEL_GRANULARITY(reg) \
1642 AssertMsg( !pCtx->reg##Hid.Attr.n.u1Present \
1643 || ( pCtx->reg##Hid.Attr.n.u1Granularity \
1644 ? (pCtx->reg##Hid.u32Limit & 0xfff) == 0xfff \
1645 : pCtx->reg##Hid.u32Limit <= 0xfffff), \
1646 ("%#x %#x %#llx\n", pCtx->reg##Hid.u32Limit, pCtx->reg##Hid.Attr.u, pCtx->reg##Hid.u64Base))
1647 SVM_ASSERT_SEL_GRANULARITY(ss);
1648 SVM_ASSERT_SEL_GRANULARITY(cs);
1649 SVM_ASSERT_SEL_GRANULARITY(ds);
1650 SVM_ASSERT_SEL_GRANULARITY(es);
1651 SVM_ASSERT_SEL_GRANULARITY(fs);
1652 SVM_ASSERT_SEL_GRANULARITY(gs);
1653#undef SVM_ASSERT_SEL_GRANULARITY
1654
1655 /*
1656 * Correct the hidden SS DPL field. It can be wrong on certain CPUs
1657 * sometimes (seen it on AMD Fusion CPUs with 64-bit guests). The CPU
1658 * always uses the CPL field in the VMCB instead of the DPL in the hidden
1659 * SS (chapter AMD spec. 15.5.1 Basic operation).
1660 */
1661 Assert(!(pVMCB->guest.u8CPL & ~0x3));
1662 pCtx->ssHid.Attr.n.u2Dpl = pVMCB->guest.u8CPL & 0x3;
1663
1664 /*
1665 * Remaining guest CPU context: TR, IDTR, GDTR, LDTR;
1666 * must sync everything otherwise we can get out of sync when jumping back to ring-3.
1667 */
1668 SVM_READ_SELREG(LDTR, ldtr);
1669 SVM_READ_SELREG(TR, tr);
1670
1671 pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1672 pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1673
1674 pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1675 pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1676
1677 /*
1678 * No reason to sync back the CRx and DRx registers as they cannot be changed by the guest
1679 * unless in the nested paging case where CR3 & CR3 can be changed by the guest.
1680 */
1681 if ( pVM->hwaccm.s.fNestedPaging
1682 && pCtx->cr3 != pVMCB->guest.u64CR3)
1683 {
1684 CPUMSetGuestCR3(pVCpu, pVMCB->guest.u64CR3);
1685 PGMUpdateCR3(pVCpu, pVMCB->guest.u64CR3);
1686 }
1687
1688 /* Note! NOW IT'S SAFE FOR LOGGING! */
1689 VMMR0LogFlushEnable(pVCpu);
1690
1691 /* Take care of instruction fusing (sti, mov ss) (see AMD spec. 15.20.5 Interrupt Shadows) */
1692 if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1693 {
1694 Log(("uInterruptState %x rip=%RGv\n", pVMCB->ctrl.u64IntShadow, (RTGCPTR)pCtx->rip));
1695 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1696 }
1697 else
1698 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1699
1700 Log2(("exitCode = %x\n", exitCode));
1701
1702 /* Sync back DR6 as it could have been changed by hitting breakpoints. */
1703 pCtx->dr[6] = pVMCB->guest.u64DR6;
1704 /* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1705 pCtx->dr[7] = pVMCB->guest.u64DR7;
1706
1707 /* Check if an injected event was interrupted prematurely. */
1708 pVCpu->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1709 if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1710 /* we don't care about 'int xx' as the instruction will be restarted. */
1711 && pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT)
1712 {
1713 Log(("Pending inject %RX64 at %RGv exit=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitCode));
1714
1715#ifdef LOG_ENABLED
1716 SVM_EVENT Event;
1717 Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
1718
1719 if ( exitCode == SVM_EXIT_EXCEPTION_E
1720 && Event.n.u8Vector == 0xE)
1721 {
1722 Log(("Double fault!\n"));
1723 }
1724#endif
1725
1726 pVCpu->hwaccm.s.Event.fPending = true;
1727 /* Error code present? (redundant) */
1728 if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1729 pVCpu->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1730 else
1731 pVCpu->hwaccm.s.Event.errCode = 0;
1732 }
1733#ifdef VBOX_WITH_STATISTICS
1734 if (exitCode == SVM_EXIT_NPF)
1735 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
1736 else
1737 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1738#endif
1739
1740 /* Sync back the TPR if it was changed. */
1741 if (fSyncTPR)
1742 {
1743 if (pVM->hwaccm.s.fTPRPatchingActive)
1744 {
1745 if ((pCtx->msrLSTAR & 0xff) != u8LastTPR)
1746 {
1747 /* Our patch code uses LSTAR for TPR caching. */
1748 rc2 = PDMApicSetTPR(pVCpu, pCtx->msrLSTAR & 0xff);
1749 AssertRC(rc2);
1750 }
1751 }
1752 else
1753 {
1754 if ((uint8_t)(u8LastTPR >> 4) != pVMCB->ctrl.IntCtrl.n.u8VTPR)
1755 {
1756 /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
1757 rc2 = PDMApicSetTPR(pVCpu, pVMCB->ctrl.IntCtrl.n.u8VTPR << 4);
1758 AssertRC(rc2);
1759 }
1760 }
1761 }
1762
1763#ifdef DBGFTRACE_ENABLED /** @todo DTrace */
1764 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x at %04:%08RX64 %RX64 %RX64 %RX64",
1765 exitCode, pCtx->cs, pCtx->rip,
1766 pVMCB->ctrl.u64ExitInfo1, pVMCB->ctrl.u64ExitInfo2, pVMCB->ctrl.ExitIntInfo.au64[0]);
1767#endif
1768#if ARCH_BITS == 64 /* for the time being */
1769 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, exitCode, pVMCB->ctrl.u64ExitInfo1, pVMCB->ctrl.u64ExitInfo2,
1770 pVMCB->ctrl.ExitIntInfo.au64[0], UINT64_MAX);
1771#endif
1772 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
1773
1774 /* Deal with the reason of the VM-exit. */
1775 switch (exitCode)
1776 {
1777 case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1778 case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1779 case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1780 case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1781 case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1782 case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1783 case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1784 case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1785 {
1786 /* Pending trap. */
1787 SVM_EVENT Event;
1788 uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1789
1790 Log2(("Hardware/software interrupt %d\n", vector));
1791 switch (vector)
1792 {
1793 case X86_XCPT_DB:
1794 {
1795 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
1796
1797 /* Note that we don't support guest and host-initiated debugging at the same time. */
1798 Assert(DBGFIsStepping(pVCpu) || CPUMIsHyperDebugStateActive(pVCpu));
1799
1800 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1801 if (rc == VINF_EM_RAW_GUEST_TRAP)
1802 {
1803 Log(("Trap %x (debug) at %016RX64\n", vector, pCtx->rip));
1804
1805 /* Reinject the exception. */
1806 Event.au64[0] = 0;
1807 Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1808 Event.n.u1Valid = 1;
1809 Event.n.u8Vector = X86_XCPT_DB;
1810
1811 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1812 goto ResumeExecution;
1813 }
1814 /* Return to ring 3 to deal with the debug exit code. */
1815 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
1816 break;
1817 }
1818
1819 case X86_XCPT_NM:
1820 {
1821 Log(("#NM fault at %RGv\n", (RTGCPTR)pCtx->rip));
1822
1823 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1824 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1825 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
1826 if (rc == VINF_SUCCESS)
1827 {
1828 Assert(CPUMIsGuestFPUStateActive(pVCpu));
1829 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
1830
1831 /* Continue execution. */
1832 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
1833
1834 goto ResumeExecution;
1835 }
1836
1837 Log(("Forward #NM fault to the guest\n"));
1838 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
1839
1840 Event.au64[0] = 0;
1841 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1842 Event.n.u1Valid = 1;
1843 Event.n.u8Vector = X86_XCPT_NM;
1844
1845 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1846 goto ResumeExecution;
1847 }
1848
1849 case X86_XCPT_PF: /* Page fault */
1850 {
1851 uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1852 RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1853
1854#ifdef VBOX_ALWAYS_TRAP_PF
1855 if (pVM->hwaccm.s.fNestedPaging)
1856 {
1857 /*
1858 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
1859 */
1860 Log(("Guest page fault at %04X:%RGv cr2=%RGv error code %x rsp=%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip,
1861 uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1862 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1863
1864 /* Now we must update CR2. */
1865 pCtx->cr2 = uFaultAddress;
1866
1867 Event.au64[0] = 0;
1868 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1869 Event.n.u1Valid = 1;
1870 Event.n.u8Vector = X86_XCPT_PF;
1871 Event.n.u1ErrorCodeValid = 1;
1872 Event.n.u32ErrorCode = errCode;
1873
1874 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1875 goto ResumeExecution;
1876 }
1877#endif
1878 Assert(!pVM->hwaccm.s.fNestedPaging);
1879
1880#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
1881 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
1882 if ( pVM->hwaccm.s.fTRPPatchingAllowed
1883 && (uFaultAddress & 0xfff) == 0x080
1884 && !(errCode & X86_TRAP_PF_P) /* not present */
1885 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
1886 && !CPUMIsGuestInLongModeEx(pCtx)
1887 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
1888 {
1889 RTGCPHYS GCPhysApicBase, GCPhys;
1890 PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
1891 GCPhysApicBase &= PAGE_BASE_GC_MASK;
1892
1893 rc = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL, &GCPhys);
1894 if ( rc == VINF_SUCCESS
1895 && GCPhys == GCPhysApicBase)
1896 {
1897 /* Only attempt to patch the instruction once. */
1898 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
1899 if (!pPatch)
1900 {
1901 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
1902 break;
1903 }
1904 }
1905 }
1906#endif
1907
1908 Log2(("Page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1909 /* Exit qualification contains the linear address of the page fault. */
1910 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
1911 TRPMSetErrorCode(pVCpu, errCode);
1912 TRPMSetFaultAddress(pVCpu, uFaultAddress);
1913
1914 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1915 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1916 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
1917 if (rc == VINF_SUCCESS)
1918 {
1919 /* We've successfully synced our shadow pages, so let's just continue execution. */
1920 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1921 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1922
1923 TRPMResetTrap(pVCpu);
1924 goto ResumeExecution;
1925 }
1926 else if (rc == VINF_EM_RAW_GUEST_TRAP)
1927 {
1928 /*
1929 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
1930 */
1931 Log2(("Forward page fault to the guest\n"));
1932 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1933 /* The error code might have been changed. */
1934 errCode = TRPMGetErrorCode(pVCpu);
1935
1936 TRPMResetTrap(pVCpu);
1937
1938 /* Now we must update CR2. */
1939 pCtx->cr2 = uFaultAddress;
1940
1941 Event.au64[0] = 0;
1942 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1943 Event.n.u1Valid = 1;
1944 Event.n.u8Vector = X86_XCPT_PF;
1945 Event.n.u1ErrorCodeValid = 1;
1946 Event.n.u32ErrorCode = errCode;
1947
1948 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1949 goto ResumeExecution;
1950 }
1951#ifdef VBOX_STRICT
1952 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
1953 LogFlow(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
1954#endif
1955 /* Need to go back to the recompiler to emulate the instruction. */
1956 TRPMResetTrap(pVCpu);
1957 break;
1958 }
1959
1960 case X86_XCPT_MF: /* Floating point exception. */
1961 {
1962 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
1963 if (!(pCtx->cr0 & X86_CR0_NE))
1964 {
1965 /* old style FPU error reporting needs some extra work. */
1966 /** @todo don't fall back to the recompiler, but do it manually. */
1967 rc = VINF_EM_RAW_EMULATE_INSTR;
1968 break;
1969 }
1970 Log(("Trap %x at %RGv\n", vector, (RTGCPTR)pCtx->rip));
1971
1972 Event.au64[0] = 0;
1973 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1974 Event.n.u1Valid = 1;
1975 Event.n.u8Vector = X86_XCPT_MF;
1976
1977 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
1978 goto ResumeExecution;
1979 }
1980
1981#ifdef VBOX_STRICT
1982 case X86_XCPT_BP: /* Breakpoint. */
1983 case X86_XCPT_GP: /* General protection failure exception.*/
1984 case X86_XCPT_UD: /* Unknown opcode exception. */
1985 case X86_XCPT_DE: /* Divide error. */
1986 case X86_XCPT_SS: /* Stack segment exception. */
1987 case X86_XCPT_NP: /* Segment not present exception. */
1988 {
1989 Event.au64[0] = 0;
1990 Event.n.u3Type = SVM_EVENT_EXCEPTION;
1991 Event.n.u1Valid = 1;
1992 Event.n.u8Vector = vector;
1993
1994 switch (vector)
1995 {
1996 case X86_XCPT_GP:
1997 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
1998 Event.n.u1ErrorCodeValid = 1;
1999 Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2000 break;
2001 case X86_XCPT_BP:
2002 /** Saves the wrong EIP on the stack (pointing to the int3 instead of the next instruction. */
2003 break;
2004 case X86_XCPT_DE:
2005 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2006 break;
2007 case X86_XCPT_UD:
2008 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2009 break;
2010 case X86_XCPT_SS:
2011 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2012 Event.n.u1ErrorCodeValid = 1;
2013 Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2014 break;
2015 case X86_XCPT_NP:
2016 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2017 Event.n.u1ErrorCodeValid = 1;
2018 Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2019 break;
2020 }
2021 Log(("Trap %x at %04x:%RGv esi=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, pCtx->esi));
2022 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2023 goto ResumeExecution;
2024 }
2025#endif
2026 default:
2027 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2028 rc = VERR_HMSVM_UNEXPECTED_XCPT_EXIT;
2029 break;
2030
2031 } /* switch (vector) */
2032 break;
2033 }
2034
2035 case SVM_EXIT_NPF:
2036 {
2037 /* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
2038 uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
2039 RTGCPHYS GCPhysFault = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
2040 PGMMODE enmShwPagingMode;
2041
2042 Assert(pVM->hwaccm.s.fNestedPaging);
2043 LogFlow(("Nested page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, GCPhysFault, errCode));
2044
2045#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2046 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2047 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2048 && (GCPhysFault & PAGE_OFFSET_MASK) == 0x080
2049 && ( !(errCode & X86_TRAP_PF_P) /* not present */
2050 || (errCode & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P | X86_TRAP_PF_RSVD) /* mmio optimization */)
2051 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2052 && !CPUMIsGuestInLongModeEx(pCtx)
2053 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2054 {
2055 RTGCPHYS GCPhysApicBase;
2056 PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
2057 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2058
2059 if (GCPhysFault == GCPhysApicBase + 0x80)
2060 {
2061 /* Only attempt to patch the instruction once. */
2062 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2063 if (!pPatch)
2064 {
2065 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2066 break;
2067 }
2068 }
2069 }
2070#endif
2071
2072 /* Handle the pagefault trap for the nested shadow table. */
2073#if HC_ARCH_BITS == 32 /** @todo shadow this in a variable. */
2074 if (CPUMIsGuestInLongModeEx(pCtx))
2075 enmShwPagingMode = PGMMODE_AMD64_NX;
2076 else
2077#endif
2078 enmShwPagingMode = PGMGetHostMode(pVM);
2079
2080 /* MMIO optimization */
2081 Assert((errCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
2082 if ((errCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
2083 {
2084 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmShwPagingMode, CPUMCTX2CORE(pCtx), GCPhysFault, errCode);
2085
2086 /*
2087 * If we succeed, resume execution.
2088 * Or, if fail in interpreting the instruction because we couldn't get the guest physical address
2089 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
2090 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
2091 * weird case. See #6043.
2092 */
2093 if ( rc == VINF_SUCCESS
2094 || rc == VERR_PAGE_TABLE_NOT_PRESENT
2095 || rc == VERR_PAGE_NOT_PRESENT)
2096 {
2097 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhysFault, (RTGCPTR)pCtx->rip));
2098 goto ResumeExecution;
2099 }
2100 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhysFault, (RTGCPTR)pCtx->rip));
2101 break;
2102 }
2103
2104 /* Exit qualification contains the linear address of the page fault. */
2105 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2106 TRPMSetErrorCode(pVCpu, errCode);
2107 TRPMSetFaultAddress(pVCpu, GCPhysFault);
2108
2109 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmShwPagingMode, errCode, CPUMCTX2CORE(pCtx), GCPhysFault);
2110 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2111
2112 /*
2113 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, #6043.
2114 */
2115 if ( rc == VINF_SUCCESS
2116 || rc == VERR_PAGE_TABLE_NOT_PRESENT
2117 || rc == VERR_PAGE_NOT_PRESENT)
2118 {
2119 /* We've successfully synced our shadow pages, so let's just continue execution. */
2120 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, GCPhysFault, errCode));
2121 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2122
2123 TRPMResetTrap(pVCpu);
2124 goto ResumeExecution;
2125 }
2126
2127#ifdef VBOX_STRICT
2128 if (rc != VINF_EM_RAW_EMULATE_INSTR)
2129 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
2130#endif
2131 /* Need to go back to the recompiler to emulate the instruction. */
2132 TRPMResetTrap(pVCpu);
2133 break;
2134 }
2135
2136 case SVM_EXIT_VINTR:
2137 /* A virtual interrupt is about to be delivered, which means IF=1. */
2138 Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
2139 pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
2140 pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
2141 goto ResumeExecution;
2142
2143 case SVM_EXIT_FERR_FREEZE:
2144 case SVM_EXIT_INTR:
2145 case SVM_EXIT_NMI:
2146 case SVM_EXIT_SMI:
2147 case SVM_EXIT_INIT:
2148 /* External interrupt; leave to allow it to be dispatched again. */
2149 rc = VINF_EM_RAW_INTERRUPT;
2150 break;
2151
2152 case SVM_EXIT_WBINVD:
2153 case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
2154 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2155 /* Skip instruction and continue directly. */
2156 pCtx->rip += 2; /* Note! hardcoded opcode size! */
2157 /* Continue execution.*/
2158 goto ResumeExecution;
2159
2160 case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
2161 {
2162 Log2(("SVM: Cpuid at %RGv for %x\n", (RTGCPTR)pCtx->rip, pCtx->eax));
2163 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2164 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2165 if (rc == VINF_SUCCESS)
2166 {
2167 /* Update EIP and continue execution. */
2168 pCtx->rip += 2; /* Note! hardcoded opcode size! */
2169 goto ResumeExecution;
2170 }
2171 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2172 rc = VINF_EM_RAW_EMULATE_INSTR;
2173 break;
2174 }
2175
2176 case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
2177 {
2178 Log2(("SVM: Rdtsc\n"));
2179 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2180 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2181 if (rc == VINF_SUCCESS)
2182 {
2183 /* Update EIP and continue execution. */
2184 pCtx->rip += 2; /* Note! hardcoded opcode size! */
2185 goto ResumeExecution;
2186 }
2187 rc = VINF_EM_RAW_EMULATE_INSTR;
2188 break;
2189 }
2190
2191 case SVM_EXIT_RDPMC: /* Guest software attempted to execute RDPMC. */
2192 {
2193 Log2(("SVM: Rdpmc %x\n", pCtx->ecx));
2194 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
2195 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2196 if (rc == VINF_SUCCESS)
2197 {
2198 /* Update EIP and continue execution. */
2199 pCtx->rip += 2; /* Note! hardcoded opcode size! */
2200 goto ResumeExecution;
2201 }
2202 rc = VINF_EM_RAW_EMULATE_INSTR;
2203 break;
2204 }
2205
2206 case SVM_EXIT_RDTSCP: /* Guest software attempted to execute RDTSCP. */
2207 {
2208 Log2(("SVM: Rdtscp\n"));
2209 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2210 rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
2211 if (rc == VINF_SUCCESS)
2212 {
2213 /* Update EIP and continue execution. */
2214 pCtx->rip += 3; /* Note! hardcoded opcode size! */
2215 goto ResumeExecution;
2216 }
2217 AssertMsgFailed(("EMU: rdtscp failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2218 rc = VINF_EM_RAW_EMULATE_INSTR;
2219 break;
2220 }
2221
2222 case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
2223 {
2224 Log2(("SVM: invlpg\n"));
2225 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
2226
2227 Assert(!pVM->hwaccm.s.fNestedPaging);
2228
2229 /* Truly a pita. Why can't SVM give the same information as VT-x? */
2230 rc = hmR0SvmInterpretInvpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
2231 if (rc == VINF_SUCCESS)
2232 {
2233 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageInvlpg);
2234 goto ResumeExecution; /* eip already updated */
2235 }
2236 break;
2237 }
2238
2239 case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
2240 case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
2241 case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
2242 case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
2243 {
2244 Log2(("SVM: %RGv mov cr%d, \n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
2245 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[exitCode - SVM_EXIT_WRITE_CR0]);
2246 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2247
2248 switch (exitCode - SVM_EXIT_WRITE_CR0)
2249 {
2250 case 0:
2251 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2252 break;
2253 case 2:
2254 break;
2255 case 3:
2256 Assert(!pVM->hwaccm.s.fNestedPaging);
2257 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
2258 break;
2259 case 4:
2260 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
2261 break;
2262 case 8:
2263 break;
2264 default:
2265 AssertFailed();
2266 }
2267 if (rc == VINF_SUCCESS)
2268 {
2269 /* EIP has been updated already. */
2270 /* Only resume if successful. */
2271 goto ResumeExecution;
2272 }
2273 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
2274 break;
2275 }
2276
2277 case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
2278 case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
2279 case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
2280 case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
2281 {
2282 Log2(("SVM: %RGv mov x, cr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
2283 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[exitCode - SVM_EXIT_READ_CR0]);
2284 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2285 if (rc == VINF_SUCCESS)
2286 {
2287 /* EIP has been updated already. */
2288 /* Only resume if successful. */
2289 goto ResumeExecution;
2290 }
2291 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
2292 break;
2293 }
2294
2295 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
2296 case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
2297 case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
2298 case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
2299 {
2300 Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
2301 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2302
2303 if ( !DBGFIsStepping(pVCpu)
2304 && !CPUMIsHyperDebugStateActive(pVCpu))
2305 {
2306 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2307
2308 /* Disable drx move intercepts. */
2309 pVMCB->ctrl.u16InterceptRdDRx = 0;
2310 pVMCB->ctrl.u16InterceptWrDRx = 0;
2311
2312 /* Save the host and load the guest debug state. */
2313 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
2314 AssertRC(rc2);
2315 goto ResumeExecution;
2316 }
2317
2318 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2319 if (rc == VINF_SUCCESS)
2320 {
2321 /* EIP has been updated already. */
2322 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
2323
2324 /* Only resume if successful. */
2325 goto ResumeExecution;
2326 }
2327 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
2328 break;
2329 }
2330
2331 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
2332 case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
2333 case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
2334 case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
2335 {
2336 Log2(("SVM: %RGv mov x, dr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
2337 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2338
2339 if (!DBGFIsStepping(pVCpu))
2340 {
2341 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2342
2343 /* Disable DRx move intercepts. */
2344 pVMCB->ctrl.u16InterceptRdDRx = 0;
2345 pVMCB->ctrl.u16InterceptWrDRx = 0;
2346
2347 /* Save the host and load the guest debug state. */
2348 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
2349 AssertRC(rc2);
2350 goto ResumeExecution;
2351 }
2352
2353 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2354 if (rc == VINF_SUCCESS)
2355 {
2356 /* EIP has been updated already. */
2357 /* Only resume if successful. */
2358 goto ResumeExecution;
2359 }
2360 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
2361 break;
2362 }
2363
2364 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2365 case SVM_EXIT_IOIO: /* I/O instruction. */
2366 {
2367 SVM_IOIO_EXIT IoExitInfo;
2368 uint32_t uIOSize, uAndVal;
2369
2370 IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
2371
2372 /** @todo could use a lookup table here */
2373 if (IoExitInfo.n.u1OP8)
2374 {
2375 uIOSize = 1;
2376 uAndVal = 0xff;
2377 }
2378 else if (IoExitInfo.n.u1OP16)
2379 {
2380 uIOSize = 2;
2381 uAndVal = 0xffff;
2382 }
2383 else if (IoExitInfo.n.u1OP32)
2384 {
2385 uIOSize = 4;
2386 uAndVal = 0xffffffff;
2387 }
2388 else
2389 {
2390 AssertFailed(); /* should be fatal. */
2391 rc = VINF_EM_RAW_EMULATE_INSTR;
2392 break;
2393 }
2394
2395 if (IoExitInfo.n.u1STR)
2396 {
2397 /* ins/outs */
2398 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
2399
2400 /* Disassemble manually to deal with segment prefixes. */
2401 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
2402 if (rc == VINF_SUCCESS)
2403 {
2404 if (IoExitInfo.n.u1Type == 0)
2405 {
2406 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2407 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
2408 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->prefix, pDis->addrmode, uIOSize);
2409 }
2410 else
2411 {
2412 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2413 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
2414 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->prefix, pDis->addrmode, uIOSize);
2415 }
2416 }
2417 else
2418 rc = VINF_EM_RAW_EMULATE_INSTR;
2419 }
2420 else
2421 {
2422 /* Normal in/out */
2423 Assert(!IoExitInfo.n.u1REP);
2424
2425 if (IoExitInfo.n.u1Type == 0)
2426 {
2427 Log2(("IOMIOPortWrite %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal,
2428 uIOSize));
2429 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
2430 rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
2431 if (rc == VINF_IOM_R3_IOPORT_WRITE)
2432 {
2433 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pVMCB->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port,
2434 uAndVal, uIOSize);
2435 }
2436 }
2437 else
2438 {
2439 uint32_t u32Val = 0;
2440
2441 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
2442 rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
2443 if (IOM_SUCCESS(rc))
2444 {
2445 /* Write back to the EAX register. */
2446 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
2447 Log2(("IOMIOPortRead %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal,
2448 uIOSize));
2449 }
2450 else if (rc == VINF_IOM_R3_IOPORT_READ)
2451 {
2452 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVMCB->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port,
2453 uAndVal, uIOSize);
2454 }
2455 }
2456 }
2457
2458 /*
2459 * Handled the I/O return codes.
2460 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2461 */
2462 if (IOM_SUCCESS(rc))
2463 {
2464 /* Update EIP and continue execution. */
2465 pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
2466 if (RT_LIKELY(rc == VINF_SUCCESS))
2467 {
2468 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2469 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2470 {
2471 /* IO operation lookup arrays. */
2472 static uint32_t const aIOSize[4] = { 1, 2, 0, 4 };
2473
2474 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
2475 for (unsigned i = 0; i < 4; i++)
2476 {
2477 unsigned uBPLen = aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2478
2479 if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
2480 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
2481 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2482 {
2483 SVM_EVENT Event;
2484
2485 Assert(CPUMIsGuestDebugStateActive(pVCpu));
2486
2487 /* Clear all breakpoint status flags and set the one we just hit. */
2488 pCtx->dr[6] &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
2489 pCtx->dr[6] |= (uint64_t)RT_BIT(i);
2490
2491 /*
2492 * Note: AMD64 Architecture Programmer's Manual 13.1:
2493 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared
2494 * by software after the contents have been read.
2495 */
2496 pVMCB->guest.u64DR6 = pCtx->dr[6];
2497
2498 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2499 pCtx->dr[7] &= ~X86_DR7_GD;
2500
2501 /* Paranoia. */
2502 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2503 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2504 pCtx->dr[7] |= 0x400; /* must be one */
2505
2506 pVMCB->guest.u64DR7 = pCtx->dr[7];
2507
2508 /* Inject the exception. */
2509 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2510
2511 Event.au64[0] = 0;
2512 Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
2513 Event.n.u1Valid = 1;
2514 Event.n.u8Vector = X86_XCPT_DB;
2515
2516 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2517 goto ResumeExecution;
2518 }
2519 }
2520 }
2521 goto ResumeExecution;
2522 }
2523 Log2(("EM status from IO at %RGv %x size %d: %Rrc\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize, VBOXSTRICTRC_VAL(rc)));
2524 break;
2525 }
2526
2527#ifdef VBOX_STRICT
2528 if (rc == VINF_IOM_R3_IOPORT_READ)
2529 Assert(IoExitInfo.n.u1Type != 0);
2530 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
2531 Assert(IoExitInfo.n.u1Type == 0);
2532 else
2533 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
2534#endif
2535 Log2(("Failed IO at %RGv %x size %d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
2536 break;
2537 }
2538
2539 case SVM_EXIT_HLT:
2540 /* Check if external interrupts are pending; if so, don't switch back. */
2541 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
2542 pCtx->rip++; /* skip hlt */
2543 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
2544 goto ResumeExecution;
2545
2546 rc = VINF_EM_HALT;
2547 break;
2548
2549 case SVM_EXIT_MWAIT_UNCOND:
2550 Log2(("SVM: mwait\n"));
2551 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
2552 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2553 if ( rc == VINF_EM_HALT
2554 || rc == VINF_SUCCESS)
2555 {
2556 /* Update EIP and continue execution. */
2557 pCtx->rip += 3; /* Note: hardcoded opcode size assumption! */
2558
2559 /* Check if external interrupts are pending; if so, don't switch back. */
2560 if ( rc == VINF_SUCCESS
2561 || ( rc == VINF_EM_HALT
2562 && EMShouldContinueAfterHalt(pVCpu, pCtx))
2563 )
2564 goto ResumeExecution;
2565 }
2566 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2567 break;
2568
2569 case SVM_EXIT_MONITOR:
2570 {
2571 Log2(("SVM: monitor\n"));
2572
2573 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
2574 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2575 if (rc == VINF_SUCCESS)
2576 {
2577 /* Update EIP and continue execution. */
2578 pCtx->rip += 3; /* Note: hardcoded opcode size assumption! */
2579 goto ResumeExecution;
2580 }
2581 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
2582 break;
2583 }
2584
2585 case SVM_EXIT_VMMCALL:
2586 rc = hmR0SvmEmulateTprVMMCall(pVM, pVCpu, pCtx);
2587 if (rc == VINF_SUCCESS)
2588 {
2589 goto ResumeExecution; /* rip already updated. */
2590 }
2591 /* no break */
2592
2593 case SVM_EXIT_RSM:
2594 case SVM_EXIT_INVLPGA:
2595 case SVM_EXIT_VMRUN:
2596 case SVM_EXIT_VMLOAD:
2597 case SVM_EXIT_VMSAVE:
2598 case SVM_EXIT_STGI:
2599 case SVM_EXIT_CLGI:
2600 case SVM_EXIT_SKINIT:
2601 {
2602 /* Unsupported instructions. */
2603 SVM_EVENT Event;
2604
2605 Event.au64[0] = 0;
2606 Event.n.u3Type = SVM_EVENT_EXCEPTION;
2607 Event.n.u1Valid = 1;
2608 Event.n.u8Vector = X86_XCPT_UD;
2609
2610 Log(("Forced #UD trap at %RGv\n", (RTGCPTR)pCtx->rip));
2611 hmR0SvmInjectEvent(pVCpu, pVMCB, pCtx, &Event);
2612 goto ResumeExecution;
2613 }
2614
2615 /* Emulate in ring-3. */
2616 case SVM_EXIT_MSR:
2617 {
2618 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
2619 if ( pVM->hwaccm.s.fTPRPatchingActive
2620 && pCtx->ecx == MSR_K8_LSTAR
2621 && pVMCB->ctrl.u64ExitInfo1 == 1 /* wrmsr */)
2622 {
2623 if ((pCtx->eax & 0xff) != u8LastTPR)
2624 {
2625 Log(("SVM: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
2626
2627 /* Our patch code uses LSTAR for TPR caching. */
2628 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
2629 AssertRC(rc2);
2630 }
2631
2632 /* Skip the instruction and continue. */
2633 pCtx->rip += 2; /* wrmsr = [0F 30] */
2634
2635 /* Only resume if successful. */
2636 goto ResumeExecution;
2637 }
2638
2639 /*
2640 * The Intel spec. claims there's an REX version of RDMSR that's slightly different,
2641 * so we play safe by completely disassembling the instruction.
2642 */
2643 STAM_COUNTER_INC((pVMCB->ctrl.u64ExitInfo1 == 0) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
2644 Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
2645 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
2646 if (rc == VINF_SUCCESS)
2647 {
2648 /* EIP has been updated already. */
2649 /* Only resume if successful. */
2650 goto ResumeExecution;
2651 }
2652 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
2653 break;
2654 }
2655
2656 case SVM_EXIT_TASK_SWITCH: /* too complicated to emulate, so fall back to the recompiler */
2657 Log(("SVM_EXIT_TASK_SWITCH: exit2=%RX64\n", pVMCB->ctrl.u64ExitInfo2));
2658 if ( !(pVMCB->ctrl.u64ExitInfo2 & (SVM_EXIT2_TASK_SWITCH_IRET | SVM_EXIT2_TASK_SWITCH_JMP))
2659 && pVCpu->hwaccm.s.Event.fPending)
2660 {
2661 SVM_EVENT Event;
2662 Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
2663
2664 /* Caused by an injected interrupt. */
2665 pVCpu->hwaccm.s.Event.fPending = false;
2666 switch (Event.n.u3Type)
2667 {
2668 case SVM_EVENT_EXTERNAL_IRQ:
2669 case SVM_EVENT_NMI:
2670 Log(("SVM_EXIT_TASK_SWITCH: reassert trap %d\n", Event.n.u8Vector));
2671 Assert(!Event.n.u1ErrorCodeValid);
2672 rc2 = TRPMAssertTrap(pVCpu, Event.n.u8Vector, TRPM_HARDWARE_INT);
2673 AssertRC(rc2);
2674 break;
2675
2676 default:
2677 /* Exceptions and software interrupts can just be restarted. */
2678 break;
2679 }
2680 }
2681 rc = VERR_EM_INTERPRETER;
2682 break;
2683
2684 case SVM_EXIT_PAUSE:
2685 case SVM_EXIT_MWAIT_ARMED:
2686 rc = VERR_EM_INTERPRETER;
2687 break;
2688
2689 case SVM_EXIT_SHUTDOWN:
2690 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2691 break;
2692
2693 case SVM_EXIT_IDTR_READ:
2694 case SVM_EXIT_GDTR_READ:
2695 case SVM_EXIT_LDTR_READ:
2696 case SVM_EXIT_TR_READ:
2697 case SVM_EXIT_IDTR_WRITE:
2698 case SVM_EXIT_GDTR_WRITE:
2699 case SVM_EXIT_LDTR_WRITE:
2700 case SVM_EXIT_TR_WRITE:
2701 case SVM_EXIT_CR0_SEL_WRITE:
2702 default:
2703 /* Unexpected exit codes. */
2704 rc = VERR_HMSVM_UNEXPECTED_EXIT;
2705 AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2706 break;
2707 }
2708
2709end:
2710
2711 /*
2712 * We are now going back to ring-3, so clear the forced action flag.
2713 */
2714 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2715
2716 /*
2717 * Signal changes to the recompiler.
2718 */
2719 CPUMSetChangedFlags(pVCpu,
2720 CPUM_CHANGED_SYSENTER_MSR
2721 | CPUM_CHANGED_LDTR
2722 | CPUM_CHANGED_GDTR
2723 | CPUM_CHANGED_IDTR
2724 | CPUM_CHANGED_TR
2725 | CPUM_CHANGED_HIDDEN_SEL_REGS);
2726
2727 /*
2728 * If we executed vmrun and an external IRQ was pending, then we don't have to do a full sync the next time.
2729 */
2730 if (exitCode == SVM_EXIT_INTR)
2731 {
2732 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
2733 /* On the next entry we'll only sync the host context. */
2734 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
2735 }
2736 else
2737 {
2738 /* On the next entry we'll sync everything. */
2739 /** @todo we can do better than this */
2740 /* Not in the VINF_PGM_CHANGE_MODE though! */
2741 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
2742 }
2743
2744 /* Translate into a less severe return code */
2745 if (rc == VERR_EM_INTERPRETER)
2746 rc = VINF_EM_RAW_EMULATE_INSTR;
2747
2748 /* Just set the correct state here instead of trying to catch every goto above. */
2749 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
2750
2751#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2752 /* Restore interrupts if we exitted after disabling them. */
2753 if (uOldEFlags != ~(RTCCUINTREG)0)
2754 ASMSetFlags(uOldEFlags);
2755#endif
2756
2757 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
2758 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2759 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2760 return VBOXSTRICTRC_TODO(rc);
2761}
2762
2763
2764/**
2765 * Emulate simple mov tpr instruction.
2766 *
2767 * @returns VBox status code.
2768 * @param pVM Pointer to the VM.
2769 * @param pVCpu The VM CPU to operate on.
2770 * @param pCtx Pointer to the guest CPU context.
2771 */
2772static int hmR0SvmEmulateTprVMMCall(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2773{
2774 int rc;
2775
2776 LogFlow(("Emulated VMMCall TPR access replacement at %RGv\n", pCtx->rip));
2777
2778 for (;;)
2779 {
2780 bool fPending;
2781 uint8_t u8Tpr;
2782
2783 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2784 if (!pPatch)
2785 break;
2786
2787 switch(pPatch->enmType)
2788 {
2789 case HWACCMTPRINSTR_READ:
2790 /* TPR caching in CR8 */
2791 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPending);
2792 AssertRC(rc);
2793
2794 rc = DISWriteReg32(CPUMCTX2CORE(pCtx), pPatch->uDstOperand, u8Tpr);
2795 AssertRC(rc);
2796
2797 LogFlow(("Emulated read successfully\n"));
2798 pCtx->rip += pPatch->cbOp;
2799 break;
2800
2801 case HWACCMTPRINSTR_WRITE_REG:
2802 case HWACCMTPRINSTR_WRITE_IMM:
2803 /* Fetch the new TPR value */
2804 if (pPatch->enmType == HWACCMTPRINSTR_WRITE_REG)
2805 {
2806 uint32_t val;
2807
2808 rc = DISFetchReg32(CPUMCTX2CORE(pCtx), pPatch->uSrcOperand, &val);
2809 AssertRC(rc);
2810 u8Tpr = val;
2811 }
2812 else
2813 u8Tpr = (uint8_t)pPatch->uSrcOperand;
2814
2815 rc = PDMApicSetTPR(pVCpu, u8Tpr);
2816 AssertRC(rc);
2817 LogFlow(("Emulated write successfully\n"));
2818 pCtx->rip += pPatch->cbOp;
2819 break;
2820
2821 default:
2822 AssertMsgFailedReturn(("Unexpected type %d\n", pPatch->enmType), VERR_HMSVM_UNEXPECTED_PATCH_TYPE);
2823 }
2824 }
2825 return VINF_SUCCESS;
2826}
2827
2828
2829/**
2830 * Enters the AMD-V session.
2831 *
2832 * @returns VBox status code.
2833 * @param pVM Pointer to the VM.
2834 * @param pVCpu The VM CPU to operate on.
2835 * @param pCpu Pointer to the CPU info struct.
2836 */
2837VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
2838{
2839 Assert(pVM->hwaccm.s.svm.fSupported);
2840
2841 LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVCpu->hwaccm.s.idLastCpu, pVCpu->hwaccm.s.uCurrentASID));
2842 pVCpu->hwaccm.s.fResumeVM = false;
2843
2844 /* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2845 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_LDTR;
2846
2847 return VINF_SUCCESS;
2848}
2849
2850
2851/**
2852 * Leaves the AMD-V session.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM Pointer to the VM.
2856 * @param pVCpu The VM CPU to operate on.
2857 * @param pCtx Pointer to the guest CPU context.
2858 */
2859VMMR0DECL(int) SVMR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2860{
2861 SVM_VMCB *pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
2862
2863 Assert(pVM->hwaccm.s.svm.fSupported);
2864
2865#ifdef DEBUG
2866 if (CPUMIsHyperDebugStateActive(pVCpu))
2867 {
2868 CPUMR0LoadHostDebugState(pVM, pVCpu);
2869 }
2870 else
2871#endif
2872 /* Save the guest debug state if necessary. */
2873 if (CPUMIsGuestDebugStateActive(pVCpu))
2874 {
2875 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, false /* skip DR6 */);
2876
2877 /* Intercept all DRx reads and writes again. Changed later on. */
2878 pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2879 pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2880
2881 /* Resync the debug registers the next time. */
2882 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
2883 }
2884 else
2885 Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2886
2887 return VINF_SUCCESS;
2888}
2889
2890
2891/**
2892 * Interprets INVLPG.
2893 *
2894 * @return VBox status code.
2895 * @param pVCpu Pointer to the VMCPU.
2896 * @param pCpu Pointer to the CPU info struct.
2897 * @param pRegFrame Pointer to the register frame.
2898 * @param ASID Tagged TLB id for the guest.
2899 */
2900static int hmR0svmInterpretInvlPg(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2901{
2902 OP_PARAMVAL param1;
2903 RTGCPTR addr;
2904 NOREF(uASID);
2905
2906 int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
2907 if(RT_FAILURE(rc))
2908 return VERR_EM_INTERPRETER;
2909
2910 switch(param1.type)
2911 {
2912 case PARMTYPE_IMMEDIATE:
2913 case PARMTYPE_ADDRESS:
2914 if(!(param1.flags & (PARAM_VAL32|PARAM_VAL64)))
2915 return VERR_EM_INTERPRETER;
2916 addr = param1.val.val64;
2917 break;
2918
2919 default:
2920 return VERR_EM_INTERPRETER;
2921 }
2922
2923 /** @todo is addr always a flat linear address or ds based
2924 * (in absence of segment override prefixes)????
2925 */
2926 rc = PGMInvalidatePage(pVCpu, addr);
2927 if (RT_SUCCESS(rc))
2928 return VINF_SUCCESS;
2929
2930 AssertRC(rc);
2931 return rc;
2932}
2933
2934
2935/**
2936 * Interprets INVLPG.
2937 *
2938 * @returns VBox status code.
2939 * @retval VINF_* Scheduling instructions.
2940 * @retval VERR_EM_INTERPRETER Something we can't cope with.
2941 * @retval VERR_* Fatal errors.
2942 *
2943 * @param pVM Pointer to the VM.
2944 * @param pRegFrame Pointer to the register frame.
2945 * @param ASID Tagged TLB id for the guest.
2946 *
2947 * @remarks Updates the EIP if an instruction was executed successfully.
2948 */
2949static int hmR0SvmInterpretInvpg(PVM pVM, PVMCPU pVCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2950{
2951 /*
2952 * Only allow 32 & 64 bit code.
2953 */
2954 DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVCpu, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2955 if (enmMode != CPUMODE_16BIT)
2956 {
2957 RTGCPTR pbCode;
2958 int rc = SELMValidateAndConvertCSAddr(pVCpu, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs,
2959 &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2960 if (RT_SUCCESS(rc))
2961 {
2962 uint32_t cbOp;
2963 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
2964
2965 pDis->mode = enmMode;
2966 rc = EMInterpretDisasOneEx(pVM, pVCpu, pbCode, pRegFrame, pDis, &cbOp);
2967 Assert(RT_FAILURE(rc) || pDis->pCurInstr->opcode == OP_INVLPG);
2968 if (RT_SUCCESS(rc) && pDis->pCurInstr->opcode == OP_INVLPG)
2969 {
2970 Assert(cbOp == pDis->opsize);
2971 rc = hmR0svmInterpretInvlPg(pVCpu, pDis, pRegFrame, uASID);
2972 if (RT_SUCCESS(rc))
2973 pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2974
2975 return rc;
2976 }
2977 }
2978 }
2979 return VERR_EM_INTERPRETER;
2980}
2981
2982
2983/**
2984 * Invalidates a guest page by guest virtual address.
2985 *
2986 * @returns VBox status code.
2987 * @param pVM Pointer to the VM.
2988 * @param pVCpu The VM CPU to operate on.
2989 * @param GCVirt Guest virtual address of the page to invalidate.
2990 */
2991VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
2992{
2993 bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB | VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
2994
2995 /* Skip it if a TLB flush is already pending. */
2996 if (!fFlushPending)
2997 {
2998 SVM_VMCB *pVMCB;
2999
3000 Log2(("SVMR0InvalidatePage %RGv\n", GCVirt));
3001 AssertReturn(pVM, VERR_INVALID_PARAMETER);
3002 Assert(pVM->hwaccm.s.svm.fSupported);
3003
3004 pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
3005 AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_HMSVM_INVALID_PVMCB);
3006
3007#if HC_ARCH_BITS == 32
3008 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invlpga takes only 32 bits addresses. */
3009 if (CPUMIsGuestInLongMode(pVCpu))
3010 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3011 else
3012#endif
3013 SVMR0InvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
3014 }
3015 return VINF_SUCCESS;
3016}
3017
3018
3019#if 0 /* obsolete, but left here for clarification. */
3020/**
3021 * Invalidates a guest page by physical address.
3022 *
3023 * @returns VBox status code.
3024 * @param pVM Pointer to the VM.
3025 * @param pVCpu The VM CPU to operate on.
3026 * @param GCPhys Guest physical address of the page to invalidate.
3027 */
3028VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3029{
3030 Assert(pVM->hwaccm.s.fNestedPaging);
3031 /* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
3032 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3033 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBInvlpga);
3034 return VINF_SUCCESS;
3035}
3036#endif
3037
3038
3039#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3040/**
3041 * Prepares for and executes VMRUN (64-bit guests from a 32-bit host).
3042 *
3043 * @returns VBox status code.
3044 * @param pVMCBHostPhys Physical address of host VMCB.
3045 * @param pVMCBPhys Physical address of the VMCB.
3046 * @param pCtx Pointer to the guest CPU context.
3047 * @param pVM Pointer to the VM.
3048 * @param pVCpu Pointer to the VMCPU.
3049 */
3050DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
3051{
3052 uint32_t aParam[4];
3053
3054 aParam[0] = (uint32_t)(pVMCBHostPhys); /* Param 1: pVMCBHostPhys - Lo. */
3055 aParam[1] = (uint32_t)(pVMCBHostPhys >> 32); /* Param 1: pVMCBHostPhys - Hi. */
3056 aParam[2] = (uint32_t)(pVMCBPhys); /* Param 2: pVMCBPhys - Lo. */
3057 aParam[3] = (uint32_t)(pVMCBPhys >> 32); /* Param 2: pVMCBPhys - Hi. */
3058
3059 return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSVMGCVMRun64, 4, &aParam[0]);
3060}
3061
3062
3063/**
3064 * Executes the specified handler in 64-bit mode.
3065 *
3066 * @returns VBox status code.
3067 * @param pVM Pointer to the VM.
3068 * @param pVCpu Pointer to the VMCPU.
3069 * @param pCtx Pointer to the guest CPU context.
3070 * @param pfnHandler Pointer to the RC handler function.
3071 * @param cbParam Number of parameters.
3072 * @param paParam Array of 32-bit parameters.
3073 */
3074VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3075{
3076 int rc;
3077 RTHCUINTREG uOldEFlags;
3078
3079 Assert(pfnHandler);
3080
3081 /* Disable interrupts. */
3082 uOldEFlags = ASMIntDisableFlags();
3083
3084 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3085 CPUMSetHyperEIP(pVCpu, pfnHandler);
3086 for (int i=(int)cbParam-1;i>=0;i--)
3087 CPUMPushHyper(pVCpu, paParam[i]);
3088
3089 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
3090 /* Call switcher. */
3091 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
3092 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
3093
3094 ASMSetFlags(uOldEFlags);
3095 return rc;
3096}
3097
3098#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
3099
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette