HWSVMR0.cpp@ 15733

Last change on this file since 15733 was 15666, checked in by vboxsync, 16 years ago
Correct the guest cr4 value when the host isn't in PAE mode (and the guest is).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 93.6 KB

Line
1	/* $Id: HWSVMR0.cpp 15666 2008-12-18 15:18:39Z vboxsync $ */
2	/** @file
3	* HWACCM SVM - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/hwacc_svm.h>
32	#include <VBox/pgm.h>
33	#include <VBox/pdm.h>
34	#include <VBox/err.h>
35	#include <VBox/log.h>
36	#include <VBox/selm.h>
37	#include <VBox/iom.h>
38	#include <VBox/dis.h>
39	#include <VBox/dbgf.h>
40	#include <VBox/disopcode.h>
41	#include <iprt/param.h>
42	#include <iprt/assert.h>
43	#include <iprt/asm.h>
44	#include <iprt/cpuset.h>
45	#include <iprt/mp.h>
46	#include "HWSVMR0.h"
47
48	/*******************************************************************************
49	* Internal Functions *
50	*******************************************************************************/
51	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID);
52
53	/*******************************************************************************
54	* Global Variables *
55	*******************************************************************************/
56	/* IO operation lookup arrays. */
57	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
58
59	/**
60	* Sets up and activates AMD-V on the current CPU
61	*
62	* @returns VBox status code.
63	* @param pCpu CPU info struct
64	* @param pVM The VM to operate on. (can be NULL after a resume!!)
65	* @param pvPageCpu Pointer to the global cpu page
66	* @param pPageCpuPhys Physical address of the global cpu page
67	*/
68	VMMR0DECL(int) SVMR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
69	{
70	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
71	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
72
73	/* We must turn on AMD-V and setup the host state physical address, as those MSRs are per-cpu/core. */
74
75	#ifdef LOG_ENABLED
76	SUPR0Printf("SVMR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
77	#endif
78
79	/* Turn on AMD-V in the EFER MSR. */
80	uint64_t val = ASMRdMsr(MSR_K6_EFER);
81	if (!(val & MSR_K6_EFER_SVME))
82	ASMWrMsr(MSR_K6_EFER, val \| MSR_K6_EFER_SVME);
83
84	/* Write the physical page address where the CPU will store the host state while executing the VM. */
85	ASMWrMsr(MSR_K8_VM_HSAVE_PA, pPageCpuPhys);
86
87	return VINF_SUCCESS;
88	}
89
90	/**
91	* Deactivates AMD-V on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pvPageCpu Pointer to the global cpu page
96	* @param pPageCpuPhys Physical address of the global cpu page
97	*/
98	VMMR0DECL(int) SVMR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
99	{
100	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
101	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
102
103	#ifdef LOG_ENABLED
104	SUPR0Printf("SVMR0DisableCpu cpu %d\n", pCpu->idCpu);
105	#endif
106
107	/* Turn off AMD-V in the EFER MSR. */
108	uint64_t val = ASMRdMsr(MSR_K6_EFER);
109	ASMWrMsr(MSR_K6_EFER, val & ~MSR_K6_EFER_SVME);
110
111	/* Invalidate host state physical address. */
112	ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
113
114	return VINF_SUCCESS;
115	}
116
117	/**
118	* Does Ring-0 per VM AMD-V init.
119	*
120	* @returns VBox status code.
121	* @param pVM The VM to operate on.
122	*/
123	VMMR0DECL(int) SVMR0InitVM(PVM pVM)
124	{
125	int rc;
126
127	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
128	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
129	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
130
131	/* Allocate one page for the host context */
132	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjVMCBHost, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
133	if (RT_FAILURE(rc))
134	return rc;
135
136	pVM->hwaccm.s.svm.pVMCBHost = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjVMCBHost);
137	pVM->hwaccm.s.svm.pVMCBHostPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjVMCBHost, 0);
138	ASMMemZeroPage(pVM->hwaccm.s.svm.pVMCBHost);
139
140	/* Allocate 12 KB for the IO bitmap (doesn't seem to be a way to convince SVM not to use it) */
141	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjIOBitmap, 3 << PAGE_SHIFT, true /* executable R0 mapping */);
142	if (RT_FAILURE(rc))
143	return rc;
144
145	pVM->hwaccm.s.svm.pIOBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjIOBitmap);
146	pVM->hwaccm.s.svm.pIOBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjIOBitmap, 0);
147	/* Set all bits to intercept all IO accesses. */
148	ASMMemFill32(pVM->hwaccm.s.svm.pIOBitmap, PAGE_SIZE*3, 0xffffffff);
149
150	/* Allocate 8 KB for the MSR bitmap (doesn't seem to be a way to convince SVM not to use it) */
151	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.svm.pMemObjMSRBitmap, 2 << PAGE_SHIFT, true /* executable R0 mapping */);
152	if (RT_FAILURE(rc))
153	return rc;
154
155	pVM->hwaccm.s.svm.pMSRBitmap = RTR0MemObjAddress(pVM->hwaccm.s.svm.pMemObjMSRBitmap);
156	pVM->hwaccm.s.svm.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.svm.pMemObjMSRBitmap, 0);
157	/* Set all bits to intercept all MSR accesses. */
158	ASMMemFill32(pVM->hwaccm.s.svm.pMSRBitmap, PAGE_SIZE*2, 0xffffffff);
159
160	/* Erratum 170 which requires a forced TLB flush for each world switch:
161	* See http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/33610.pdf
162	*
163	* All BH-G1/2 and DH-G1/2 models include a fix:
164	* Athlon X2: 0x6b 1/2
165	* 0x68 1/2
166	* Athlon 64: 0x7f 1
167	* 0x6f 2
168	* Sempron: 0x7f 1/2
169	* 0x6f 2
170	* 0x6c 2
171	* 0x7c 2
172	* Turion 64: 0x68 2
173	*
174	*/
175	uint32_t u32Dummy;
176	uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
177	ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
178	u32BaseFamily= (u32Version >> 8) & 0xf;
179	u32Family = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
180	u32Model = ((u32Version >> 4) & 0xf);
181	u32Model = u32Model \| ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
182	u32Stepping = u32Version & 0xf;
183	if ( u32Family == 0xf
184	&& !((u32Model == 0x68 \|\| u32Model == 0x6b \|\| u32Model == 0x7f) && u32Stepping >= 1)
185	&& !((u32Model == 0x6f \|\| u32Model == 0x6c \|\| u32Model == 0x7c) && u32Stepping >= 2))
186	{
187	Log(("SVMR0InitVM: AMD cpu with erratum 170 family %x model %x stepping %x\n", u32Family, u32Model, u32Stepping));
188	pVM->hwaccm.s.svm.fAlwaysFlushTLB = true;
189	}
190
191	/* Allocate VMCBs for all guest CPUs. */
192	for (unsigned i=0;i<pVM->cCPUs;i++)
193	{
194	pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
195
196	/* Allocate one page for the VM control block (VMCB). */
197	rc = RTR0MemObjAllocCont(&pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
198	if (RT_FAILURE(rc))
199	return rc;
200
201	pVM->aCpus[i].hwaccm.s.svm.pVMCB = RTR0MemObjAddress(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB);
202	pVM->aCpus[i].hwaccm.s.svm.pVMCBPhys = RTR0MemObjGetPagePhysAddr(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, 0);
203	ASMMemZeroPage(pVM->aCpus[i].hwaccm.s.svm.pVMCB);
204	}
205
206	return VINF_SUCCESS;
207	}
208
209	/**
210	* Does Ring-0 per VM AMD-V termination.
211	*
212	* @returns VBox status code.
213	* @param pVM The VM to operate on.
214	*/
215	VMMR0DECL(int) SVMR0TermVM(PVM pVM)
216	{
217	for (unsigned i=0;i<pVM->cCPUs;i++)
218	{
219	if (pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB != NIL_RTR0MEMOBJ)
220	{
221	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB, false);
222	pVM->aCpus[i].hwaccm.s.svm.pVMCB = 0;
223	pVM->aCpus[i].hwaccm.s.svm.pVMCBPhys = 0;
224	pVM->aCpus[i].hwaccm.s.svm.pMemObjVMCB = NIL_RTR0MEMOBJ;
225	}
226	}
227	if (pVM->hwaccm.s.svm.pMemObjVMCBHost != NIL_RTR0MEMOBJ)
228	{
229	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjVMCBHost, false);
230	pVM->hwaccm.s.svm.pVMCBHost = 0;
231	pVM->hwaccm.s.svm.pVMCBHostPhys = 0;
232	pVM->hwaccm.s.svm.pMemObjVMCBHost = NIL_RTR0MEMOBJ;
233	}
234	if (pVM->hwaccm.s.svm.pMemObjIOBitmap != NIL_RTR0MEMOBJ)
235	{
236	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjIOBitmap, false);
237	pVM->hwaccm.s.svm.pIOBitmap = 0;
238	pVM->hwaccm.s.svm.pIOBitmapPhys = 0;
239	pVM->hwaccm.s.svm.pMemObjIOBitmap = NIL_RTR0MEMOBJ;
240	}
241	if (pVM->hwaccm.s.svm.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
242	{
243	RTR0MemObjFree(pVM->hwaccm.s.svm.pMemObjMSRBitmap, false);
244	pVM->hwaccm.s.svm.pMSRBitmap = 0;
245	pVM->hwaccm.s.svm.pMSRBitmapPhys = 0;
246	pVM->hwaccm.s.svm.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
247	}
248	return VINF_SUCCESS;
249	}
250
251	/**
252	* Sets up AMD-V for the specified VM
253	*
254	* @returns VBox status code.
255	* @param pVM The VM to operate on.
256	*/
257	VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
258	{
259	int rc = VINF_SUCCESS;
260	SVM_VMCB *pVMCB;
261
262	AssertReturn(pVM, VERR_INVALID_PARAMETER);
263
264	Assert(pVM->hwaccm.s.svm.fSupported);
265
266	for (unsigned i=0;i<pVM->cCPUs;i++)
267	{
268	pVMCB = (SVM_VMCB *)pVM->aCpus[i].hwaccm.s.svm.pVMCB;
269	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
270
271	/* Program the control fields. Most of them never have to be changed again. */
272	/* CR0/3/4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
273	/* Note: CR0 & CR4 can be safely read when guest and shadow copies are identical. */
274	if (!pVM->hwaccm.s.fNestedPaging)
275	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
276	else
277	pVMCB->ctrl.u16InterceptRdCRx = RT_BIT(0) \| RT_BIT(4);
278
279	/*
280	* CR0/3/4 writes must be intercepted for obvious reasons.
281	*/
282	if (!pVM->hwaccm.s.fNestedPaging)
283	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(3) \| RT_BIT(4);
284	else
285	pVMCB->ctrl.u16InterceptWrCRx = RT_BIT(0) \| RT_BIT(4) \| RT_BIT(8);
286
287	/* Intercept all DRx reads and writes by default. Changed later on. */
288	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
289	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
290
291	/* Currently we don't care about DRx reads or writes. DRx registers are trashed.
292	* All breakpoints are automatically cleared when the VM exits.
293	*/
294
295	pVMCB->ctrl.u32InterceptException = HWACCM_SVM_TRAP_MASK;
296	#ifndef DEBUG
297	if (pVM->hwaccm.s.fNestedPaging)
298	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
299	#endif
300
301	pVMCB->ctrl.u32InterceptCtrl1 = SVM_CTRL1_INTERCEPT_INTR
302	\| SVM_CTRL1_INTERCEPT_VINTR
303	\| SVM_CTRL1_INTERCEPT_NMI
304	\| SVM_CTRL1_INTERCEPT_SMI
305	\| SVM_CTRL1_INTERCEPT_INIT
306	\| SVM_CTRL1_INTERCEPT_RDPMC
307	\| SVM_CTRL1_INTERCEPT_CPUID
308	\| SVM_CTRL1_INTERCEPT_RSM
309	\| SVM_CTRL1_INTERCEPT_HLT
310	\| SVM_CTRL1_INTERCEPT_INOUT_BITMAP
311	\| SVM_CTRL1_INTERCEPT_MSR_SHADOW
312	\| SVM_CTRL1_INTERCEPT_INVLPG
313	\| SVM_CTRL1_INTERCEPT_INVLPGA /* AMD only */
314	\| SVM_CTRL1_INTERCEPT_TASK_SWITCH
315	\| SVM_CTRL1_INTERCEPT_SHUTDOWN /* fatal */
316	\| SVM_CTRL1_INTERCEPT_FERR_FREEZE; /* Legacy FPU FERR handling. */
317	;
318	/* With nested paging we don't care about invlpg anymore. */
319	if (pVM->hwaccm.s.fNestedPaging)
320	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_INVLPG;
321
322	pVMCB->ctrl.u32InterceptCtrl2 = SVM_CTRL2_INTERCEPT_VMRUN /* required */
323	\| SVM_CTRL2_INTERCEPT_VMMCALL
324	\| SVM_CTRL2_INTERCEPT_VMLOAD
325	\| SVM_CTRL2_INTERCEPT_VMSAVE
326	\| SVM_CTRL2_INTERCEPT_STGI
327	\| SVM_CTRL2_INTERCEPT_CLGI
328	\| SVM_CTRL2_INTERCEPT_SKINIT
329	\| SVM_CTRL2_INTERCEPT_WBINVD
330	\| SVM_CTRL2_INTERCEPT_MWAIT_UNCOND; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
331	;
332	Log(("pVMCB->ctrl.u32InterceptException = %x\n", pVMCB->ctrl.u32InterceptException));
333	Log(("pVMCB->ctrl.u32InterceptCtrl1 = %x\n", pVMCB->ctrl.u32InterceptCtrl1));
334	Log(("pVMCB->ctrl.u32InterceptCtrl2 = %x\n", pVMCB->ctrl.u32InterceptCtrl2));
335
336	/* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
337	pVMCB->ctrl.IntCtrl.n.u1VIrqMasking = 1;
338	/* Ignore the priority in the TPR; just deliver it when we tell it to. */
339	pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
340
341	/* Set IO and MSR bitmap addresses. */
342	pVMCB->ctrl.u64IOPMPhysAddr = pVM->hwaccm.s.svm.pIOBitmapPhys;
343	pVMCB->ctrl.u64MSRPMPhysAddr = pVM->hwaccm.s.svm.pMSRBitmapPhys;
344
345	/* No LBR virtualization. */
346	pVMCB->ctrl.u64LBRVirt = 0;
347
348	/** The ASID must start at 1; the host uses 0. */
349	pVMCB->ctrl.TLBCtrl.n.u32ASID = 1;
350
351	/** Setup the PAT msr (nested paging only) */
352	pVMCB->guest.u64GPAT = 0x0007040600070406ULL;
353	}
354	return rc;
355	}
356
357
358	/**
359	* Injects an event (trap or external interrupt)
360	*
361	* @param pVM The VM to operate on.
362	* @param pVMCB SVM control block
363	* @param pCtx CPU Context
364	* @param pIntInfo SVM interrupt info
365	*/
366	inline void SVMR0InjectEvent(PVM pVM, SVM_VMCB pVMCB, CPUMCTX pCtx, SVM_EVENT* pEvent)
367	{
368	#ifdef VBOX_STRICT
369	if (pEvent->n.u8Vector == 0xE)
370	Log(("SVM: Inject int %d at %RGv error code=%02x CR2=%RGv intInfo=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode, (RTGCPTR)pCtx->cr2, pEvent->au64[0]));
371	else
372	if (pEvent->n.u8Vector < 0x20)
373	Log(("SVM: Inject int %d at %RGv error code=%08x\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip, pEvent->n.u32ErrorCode));
374	else
375	{
376	Log(("INJ-EI: %x at %RGv\n", pEvent->n.u8Vector, (RTGCPTR)pCtx->rip));
377	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
378	Assert(pCtx->eflags.u32 & X86_EFL_IF);
379	}
380	#endif
381
382	/* Set event injection state. */
383	pVMCB->ctrl.EventInject.au64[0] = pEvent->au64[0];
384	}
385
386
387	/**
388	* Checks for pending guest interrupts and injects them
389	*
390	* @returns VBox status code.
391	* @param pVM The VM to operate on.
392	* @param pVCpu The VM CPU to operate on.
393	* @param pVMCB SVM control block
394	* @param pCtx CPU Context
395	*/
396	static int SVMR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, SVM_VMCB pVMCB, CPUMCTX pCtx)
397	{
398	int rc;
399
400	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
401	if (pVCpu->hwaccm.s.Event.fPending)
402	{
403	SVM_EVENT Event;
404
405	Log(("Reinjecting event %08x %08x at %RGv\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip));
406	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
407	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
408	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
409
410	pVCpu->hwaccm.s.Event.fPending = false;
411	return VINF_SUCCESS;
412	}
413
414	if (pVM->hwaccm.s.fInjectNMI)
415	{
416	SVM_EVENT Event;
417
418	Event.n.u8Vector = X86_XCPT_NMI;
419	Event.n.u1Valid = 1;
420	Event.n.u32ErrorCode = 0;
421	Event.n.u3Type = SVM_EVENT_NMI;
422
423	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
424	pVM->hwaccm.s.fInjectNMI = false;
425	return VINF_SUCCESS;
426	}
427
428	/* When external interrupts are pending, we should exit the VM when IF is set. */
429	if ( !TRPMHasTrap(pVM)
430	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
431	{
432	if ( !(pCtx->eflags.u32 & X86_EFL_IF)
433	\|\| VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
434	{
435	if (!pVMCB->ctrl.IntCtrl.n.u1VIrqValid)
436	{
437	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
438	LogFlow(("Enable irq window exit!\n"));
439	else
440	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS -> irq window exit\n", (RTGCPTR)pCtx->rip));
441
442	/** @todo use virtual interrupt method to inject a pending irq; dispatched as soon as guest.IF is set. */
443	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_VINTR;
444	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 1;
445	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0; /* don't care */
446	}
447	}
448	else
449	{
450	uint8_t u8Interrupt;
451
452	rc = PDMGetInterrupt(pVM, &u8Interrupt);
453	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc\n", u8Interrupt, u8Interrupt, rc));
454	if (RT_SUCCESS(rc))
455	{
456	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
457	AssertRC(rc);
458	}
459	else
460	{
461	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
462	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
463	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
464	/* Just continue */
465	}
466	}
467	}
468
469	#ifdef VBOX_STRICT
470	if (TRPMHasTrap(pVM))
471	{
472	uint8_t u8Vector;
473	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
474	AssertRC(rc);
475	}
476	#endif
477
478	if ( pCtx->eflags.u32 & X86_EFL_IF
479	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
480	&& TRPMHasTrap(pVM)
481	)
482	{
483	uint8_t u8Vector;
484	int rc;
485	TRPMEVENT enmType;
486	SVM_EVENT Event;
487	RTGCUINT u32ErrorCode;
488
489	Event.au64[0] = 0;
490
491	/* If a new event is pending, then dispatch it now. */
492	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &u32ErrorCode, 0);
493	AssertRC(rc);
494	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
495	Assert(enmType != TRPM_SOFTWARE_INT);
496
497	/* Clear the pending trap. */
498	rc = TRPMResetTrap(pVM);
499	AssertRC(rc);
500
501	Event.n.u8Vector = u8Vector;
502	Event.n.u1Valid = 1;
503	Event.n.u32ErrorCode = u32ErrorCode;
504
505	if (enmType == TRPM_TRAP)
506	{
507	switch (u8Vector) {
508	case 8:
509	case 10:
510	case 11:
511	case 12:
512	case 13:
513	case 14:
514	case 17:
515	/* Valid error codes. */
516	Event.n.u1ErrorCodeValid = 1;
517	break;
518	default:
519	break;
520	}
521	if (u8Vector == X86_XCPT_NMI)
522	Event.n.u3Type = SVM_EVENT_NMI;
523	else
524	Event.n.u3Type = SVM_EVENT_EXCEPTION;
525	}
526	else
527	Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
528
529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
530	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
531	} /* if (interrupts can be dispatched) */
532
533	return VINF_SUCCESS;
534	}
535
536	/**
537	* Save the host state
538	*
539	* @returns VBox status code.
540	* @param pVM The VM to operate on.
541	* @param pVCpu The VM CPU to operate on.
542	*/
543	VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
544	{
545	NOREF(pVM);
546	NOREF(pVCpu);
547	/* Nothing to do here. */
548	return VINF_SUCCESS;
549	}
550
551	/**
552	* Loads the guest state
553	*
554	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
555	*
556	* @returns VBox status code.
557	* @param pVM The VM to operate on.
558	* @param pVCpu The VM CPU to operate on.
559	* @param pCtx Guest context
560	*/
561	VMMR0DECL(int) SVMR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
562	{
563	RTGCUINTPTR val;
564	SVM_VMCB *pVMCB;
565
566	if (pVM == NULL)
567	return VERR_INVALID_PARAMETER;
568
569	/* Setup AMD SVM. */
570	Assert(pVM->hwaccm.s.svm.fSupported);
571
572	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
573	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
574
575	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
576	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
577	{
578	SVM_WRITE_SELREG(CS, cs);
579	SVM_WRITE_SELREG(SS, ss);
580	SVM_WRITE_SELREG(DS, ds);
581	SVM_WRITE_SELREG(ES, es);
582	SVM_WRITE_SELREG(FS, fs);
583	SVM_WRITE_SELREG(GS, gs);
584	}
585
586	/* Guest CPU context: LDTR. */
587	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
588	{
589	SVM_WRITE_SELREG(LDTR, ldtr);
590	}
591
592	/* Guest CPU context: TR. */
593	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
594	{
595	SVM_WRITE_SELREG(TR, tr);
596	}
597
598	/* Guest CPU context: GDTR. */
599	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
600	{
601	pVMCB->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
602	pVMCB->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
603	}
604
605	/* Guest CPU context: IDTR. */
606	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
607	{
608	pVMCB->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
609	pVMCB->guest.IDTR.u64Base = pCtx->idtr.pIdt;
610	}
611
612	/*
613	* Sysenter MSRs (unconditional)
614	*/
615	pVMCB->guest.u64SysEnterCS = pCtx->SysEnter.cs;
616	pVMCB->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
617	pVMCB->guest.u64SysEnterESP = pCtx->SysEnter.esp;
618
619	/* Control registers */
620	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
621	{
622	val = pCtx->cr0;
623	if (!CPUMIsGuestFPUStateActive(pVCpu))
624	{
625	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
626	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
627	}
628	else
629	{
630	/** @todo check if we support the old style mess correctly. */
631	if (!(val & X86_CR0_NE))
632	{
633	Log(("Forcing X86_CR0_NE!!!\n"));
634
635	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
636	if (!pVCpu->hwaccm.s.fFPUOldStyleOverride)
637	{
638	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_MF);
639	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
640	}
641	}
642	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
643	}
644	/* Always enable caching. */
645	val &= ~(X86_CR0_CD\|X86_CR0_NW);
646
647	/* Note: WP is not relevant in nested paging mode as we catch accesses on the (guest) physical level. */
648	/* Note: In nested paging mode the guest is allowed to run with paging disabled; the guest physical to host physical translation will remain active. */
649	if (!pVM->hwaccm.s.fNestedPaging)
650	{
651	val \|= X86_CR0_PG; /* Paging is always enabled; even when the guest is running in real mode or PE without paging. */
652	val \|= X86_CR0_WP; /* Must set this as we rely on protect various pages and supervisor writes must be caught. */
653	}
654	pVMCB->guest.u64CR0 = val;
655	}
656	/* CR2 as well */
657	pVMCB->guest.u64CR2 = pCtx->cr2;
658
659	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
660	{
661	/* Save our shadow CR3 register. */
662	if (pVM->hwaccm.s.fNestedPaging)
663	{
664	PGMMODE enmShwPagingMode;
665
666	#if HC_ARCH_BITS == 32
667	if (CPUMIsGuestInLongModeEx(pCtx))
668	enmShwPagingMode = PGMMODE_AMD64_NX;
669	else
670	#endif
671	enmShwPagingMode = PGMGetHostMode(pVM);
672
673	pVMCB->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVM, enmShwPagingMode);
674	Assert(pVMCB->ctrl.u64NestedPagingCR3);
675	pVMCB->guest.u64CR3 = pCtx->cr3;
676	}
677	else
678	{
679	pVMCB->guest.u64CR3 = PGMGetHyperCR3(pVM);
680	Assert(pVMCB->guest.u64CR3 \|\| VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL));
681	}
682	}
683
684	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
685	{
686	val = pCtx->cr4;
687	if (!pVM->hwaccm.s.fNestedPaging)
688	{
689	switch(pVCpu->hwaccm.s.enmShadowMode)
690	{
691	case PGMMODE_REAL:
692	case PGMMODE_PROTECTED: /* Protected mode, no paging. */
693	AssertFailed();
694	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
695
696	case PGMMODE_32_BIT: /* 32-bit paging. */
697	val &= ~X86_CR4_PAE;
698	break;
699
700	case PGMMODE_PAE: /* PAE paging. */
701	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
702	/** @todo use normal 32 bits paging */
703	val \|= X86_CR4_PAE;
704	break;
705
706	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
707	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
708	#ifdef VBOX_ENABLE_64_BITS_GUESTS
709	break;
710	#else
711	AssertFailed();
712	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
713	#endif
714
715	default: /* shut up gcc */
716	AssertFailed();
717	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
718	}
719	}
720	pVMCB->guest.u64CR4 = val;
721	}
722
723	/* Debug registers. */
724	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
725	{
726	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
727	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
728
729	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
730	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
731	pCtx->dr[7] \|= 0x400; /* must be one */
732
733	pVMCB->guest.u64DR7 = pCtx->dr[7];
734	pVMCB->guest.u64DR6 = pCtx->dr[6];
735
736	/* Sync the debug state now if any breakpoint is armed. */
737	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
738	&& !CPUMIsGuestDebugStateActive(pVM)
739	&& !DBGFIsStepping(pVM))
740	{
741	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
742
743	/* Disable drx move intercepts. */
744	pVMCB->ctrl.u16InterceptRdDRx = 0;
745	pVMCB->ctrl.u16InterceptWrDRx = 0;
746
747	/* Save the host and load the guest debug state. */
748	int rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
749	AssertRC(rc);
750	}
751	}
752
753	/* EIP, ESP and EFLAGS */
754	pVMCB->guest.u64RIP = pCtx->rip;
755	pVMCB->guest.u64RSP = pCtx->rsp;
756	pVMCB->guest.u64RFlags = pCtx->eflags.u32;
757
758	/* Set CPL */
759	pVMCB->guest.u8CPL = pCtx->csHid.Attr.n.u2Dpl;
760
761	/* RAX/EAX too, as VMRUN uses RAX as an implicit parameter. */
762	pVMCB->guest.u64RAX = pCtx->rax;
763
764	/* vmrun will fail without MSR_K6_EFER_SVME. */
765	pVMCB->guest.u64EFER = pCtx->msrEFER \| MSR_K6_EFER_SVME;
766
767	/* 64 bits guest mode? */
768	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
769	{
770	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
771	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
772	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
773	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
774	#else
775	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
776	if (!pVM->hwaccm.s.fAllow64BitGuests)
777	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
778	# endif
779	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun64;
780	#endif
781	/* Unconditionally update these as wrmsr might have changed them. (HWACCM_CHANGED_GUEST_SEGMENT_REGS will not be set) */
782	pVMCB->guest.FS.u64Base = pCtx->fsHid.u64Base;
783	pVMCB->guest.GS.u64Base = pCtx->gsHid.u64Base;
784	}
785	else
786	{
787	/* Filter out the MSR_K6_LME bit or else AMD-V expects amd64 shadow paging. */
788	pVMCB->guest.u64EFER &= ~MSR_K6_EFER_LME;
789
790	pVCpu->hwaccm.s.svm.pfnVMRun = SVMR0VMRun;
791	}
792
793	/* TSC offset. */
794	if (TMCpuTickCanUseRealTSC(pVM, &pVMCB->ctrl.u64TSCOffset))
795	{
796	pVMCB->ctrl.u32InterceptCtrl1 &= ~SVM_CTRL1_INTERCEPT_RDTSC;
797	pVMCB->ctrl.u32InterceptCtrl2 &= ~SVM_CTRL2_INTERCEPT_RDTSCP;
798	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
799	}
800	else
801	{
802	pVMCB->ctrl.u32InterceptCtrl1 \|= SVM_CTRL1_INTERCEPT_RDTSC;
803	pVMCB->ctrl.u32InterceptCtrl2 \|= SVM_CTRL2_INTERCEPT_RDTSCP;
804	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
805	}
806
807	/* Sync the various msrs for 64 bits mode. */
808	pVMCB->guest.u64STAR = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
809	pVMCB->guest.u64LSTAR = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
810	pVMCB->guest.u64CSTAR = pCtx->msrCSTAR; /* compatibility mode syscall rip */
811	pVMCB->guest.u64SFMASK = pCtx->msrSFMASK; /* syscall flag mask */
812	pVMCB->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
813
814	#ifdef DEBUG
815	/* Intercept X86_XCPT_DB if stepping is enabled */
816	if (DBGFIsStepping(pVM))
817	pVMCB->ctrl.u32InterceptException \|= RT_BIT(X86_XCPT_DB);
818	else
819	pVMCB->ctrl.u32InterceptException &= ~RT_BIT(X86_XCPT_DB);
820	#endif
821
822	/* Done. */
823	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
824
825	return VINF_SUCCESS;
826	}
827
828
829	/**
830	* Runs guest code in an AMD-V VM.
831	*
832	* @returns VBox status code.
833	* @param pVM The VM to operate on.
834	* @param pVCpu The VM CPU to operate on.
835	* @param pCtx Guest context
836	*/
837	VMMR0DECL(int) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
838	{
839	int rc = VINF_SUCCESS;
840	uint64_t exitCode = (uint64_t)SVM_EXIT_INVALID;
841	SVM_VMCB *pVMCB;
842	bool fSyncTPR = false;
843	unsigned cResume = 0;
844	uint8_t u8LastVTPR;
845	PHWACCM_CPUINFO pCpu = 0;
846	#ifdef VBOX_STRICT
847	RTCPUID idCpuCheck;
848	#endif
849
850	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
851
852	pVMCB = (SVM_VMCB *)pVCpu->hwaccm.s.svm.pVMCB;
853	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
854
855	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
856	*/
857	ResumeExecution:
858	Assert(!HWACCMR0SuspendPending());
859
860	/* Safety precaution; looping for too long here can have a very bad effect on the host */
861	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
862	{
863	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
864	rc = VINF_EM_RAW_INTERRUPT;
865	goto end;
866	}
867
868	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
869	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
870	{
871	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
872	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
873	{
874	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
875	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
876	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
877	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
878	*/
879	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
880	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
881	pVMCB->ctrl.u64IntShadow = 0;
882	}
883	}
884	else
885	{
886	/* Irq inhibition is no longer active; clear the corresponding SVM state. */
887	pVMCB->ctrl.u64IntShadow = 0;
888	}
889
890	/* Check for pending actions that force us to go back to ring 3. */
891	#ifdef DEBUG
892	/* Intercept X86_XCPT_DB if stepping is enabled */
893	if (!DBGFIsStepping(pVM))
894	#endif
895	{
896	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
897	{
898	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
899	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
900	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
901	rc = VINF_EM_RAW_TO_R3;
902	goto end;
903	}
904	}
905
906	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
907	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
908	{
909	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
910	rc = VINF_EM_PENDING_REQUEST;
911	goto end;
912	}
913
914	/* When external interrupts are pending, we should exit the VM when IF is set. */
915	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
916	rc = SVMR0CheckPendingInterrupt(pVM, pVCpu, pVMCB, pCtx);
917	if (RT_FAILURE(rc))
918	{
919	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
920	goto end;
921	}
922
923	/* TPR caching using CR8 is only available in 64 bits mode */
924	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
925	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!!!!! */
926	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
927	{
928	bool fPending;
929
930	/* TPR caching in CR8 */
931	int rc = PDMApicGetTPR(pVM, &u8LastVTPR, &fPending);
932	AssertRC(rc);
933	pVMCB->ctrl.IntCtrl.n.u8VTPR = u8LastVTPR;
934
935	if (fPending)
936	{
937	/* A TPR change could activate a pending interrupt, so catch cr8 writes. */
938	pVMCB->ctrl.u16InterceptWrCRx \|= RT_BIT(8);
939	}
940	else
941	/* No interrupts are pending, so we don't need to be explicitely notified.
942	* There are enough world switches for detecting pending interrupts.
943	*/
944	pVMCB->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
945
946	fSyncTPR = !fPending;
947	}
948
949	/* All done! Let's start VM execution. */
950	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, x);
951
952	/* Enable nested paging if necessary (disabled each time after #VMEXIT). */
953	pVMCB->ctrl.NestedPaging.n.u1NestedPaging = pVM->hwaccm.s.fNestedPaging;
954
955	#ifdef LOG_ENABLED
956	pCpu = HWACCMR0GetCurrentCpu();
957	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
958	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
959	{
960	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
961	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
962	else
963	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
964	}
965	if (pCpu->fFlushTLB)
966	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
967	#endif
968
969	/*
970	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
971	* (until the actual world switch)
972	*/
973
974	#ifdef VBOX_STRICT
975	idCpuCheck = RTMpCpuId();
976	#endif
977
978	/* Load the guest state; must be here as it sets up the shadow cr0 for lazy fpu syncing! */
979	rc = SVMR0LoadGuestState(pVM, pVCpu, pCtx);
980	if (rc != VINF_SUCCESS)
981	{
982	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
983	goto end;
984	}
985
986	pCpu = HWACCMR0GetCurrentCpu();
987	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
988	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
989	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
990	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
991	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
992	{
993	/* Force a TLB flush on VM entry. */
994	pVCpu->hwaccm.s.fForceTLBFlush = true;
995	}
996	else
997	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
998
999	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1000
1001	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1002	if ( pVCpu->hwaccm.s.fForceTLBFlush
1003	&& !pVM->hwaccm.s.svm.fAlwaysFlushTLB)
1004	{
1005	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1006	\|\| pCpu->fFlushTLB)
1007	{
1008	pCpu->fFlushTLB = false;
1009	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1010	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = 1; /* wrap around; flush TLB */
1011	pCpu->cTLBFlushes++;
1012	}
1013	else
1014	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1015
1016	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1017	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1018	}
1019	else
1020	{
1021	Assert(!pCpu->fFlushTLB \|\| pVM->hwaccm.s.svm.fAlwaysFlushTLB);
1022
1023	/* We never increase uCurrentASID in the fAlwaysFlushTLB (erratum 170) case. */
1024	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1025	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1026
1027	Assert(!pVM->hwaccm.s.svm.fAlwaysFlushTLB \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1028	pVMCB->ctrl.TLBCtrl.n.u1TLBFlush = pVCpu->hwaccm.s.fForceTLBFlush;
1029	}
1030	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1031	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1032	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1033	pVMCB->ctrl.TLBCtrl.n.u32ASID = pVCpu->hwaccm.s.uCurrentASID;
1034
1035	#ifdef VBOX_WITH_STATISTICS
1036	if (pVMCB->ctrl.TLBCtrl.n.u1TLBFlush)
1037	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1038	else
1039	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1040	#endif
1041
1042	/* In case we execute a goto ResumeExecution later on. */
1043	pVCpu->hwaccm.s.fResumeVM = true;
1044	pVCpu->hwaccm.s.fForceTLBFlush = pVM->hwaccm.s.svm.fAlwaysFlushTLB;
1045
1046	Assert(sizeof(pVCpu->hwaccm.s.svm.pVMCBPhys) == 8);
1047	Assert(pVMCB->ctrl.IntCtrl.n.u1VIrqMasking);
1048	Assert(pVMCB->ctrl.u64IOPMPhysAddr == pVM->hwaccm.s.svm.pIOBitmapPhys);
1049	Assert(pVMCB->ctrl.u64MSRPMPhysAddr == pVM->hwaccm.s.svm.pMSRBitmapPhys);
1050	Assert(pVMCB->ctrl.u64LBRVirt == 0);
1051
1052	#ifdef VBOX_STRICT
1053	Assert(idCpuCheck == RTMpCpuId());
1054	#endif
1055	TMNotifyStartOfExecution(pVM);
1056	pVCpu->hwaccm.s.svm.pfnVMRun(pVM->hwaccm.s.svm.pVMCBHostPhys, pVCpu->hwaccm.s.svm.pVMCBPhys, pCtx, pVM, pVCpu);
1057	TMNotifyEndOfExecution(pVM);
1058	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, x);
1059
1060	/*
1061	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1062	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1063	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1064	*/
1065
1066	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, x);
1067
1068	/* Reason for the VM exit */
1069	exitCode = pVMCB->ctrl.u64ExitCode;
1070
1071	if (exitCode == (uint64_t)SVM_EXIT_INVALID) /* Invalid guest state. */
1072	{
1073	HWACCMDumpRegs(pVM, pCtx);
1074	#ifdef DEBUG
1075	Log(("ctrl.u16InterceptRdCRx %x\n", pVMCB->ctrl.u16InterceptRdCRx));
1076	Log(("ctrl.u16InterceptWrCRx %x\n", pVMCB->ctrl.u16InterceptWrCRx));
1077	Log(("ctrl.u16InterceptRdDRx %x\n", pVMCB->ctrl.u16InterceptRdDRx));
1078	Log(("ctrl.u16InterceptWrDRx %x\n", pVMCB->ctrl.u16InterceptWrDRx));
1079	Log(("ctrl.u32InterceptException %x\n", pVMCB->ctrl.u32InterceptException));
1080	Log(("ctrl.u32InterceptCtrl1 %x\n", pVMCB->ctrl.u32InterceptCtrl1));
1081	Log(("ctrl.u32InterceptCtrl2 %x\n", pVMCB->ctrl.u32InterceptCtrl2));
1082	Log(("ctrl.u64IOPMPhysAddr %RX64\n", pVMCB->ctrl.u64IOPMPhysAddr));
1083	Log(("ctrl.u64MSRPMPhysAddr %RX64\n", pVMCB->ctrl.u64MSRPMPhysAddr));
1084	Log(("ctrl.u64TSCOffset %RX64\n", pVMCB->ctrl.u64TSCOffset));
1085
1086	Log(("ctrl.TLBCtrl.u32ASID %x\n", pVMCB->ctrl.TLBCtrl.n.u32ASID));
1087	Log(("ctrl.TLBCtrl.u1TLBFlush %x\n", pVMCB->ctrl.TLBCtrl.n.u1TLBFlush));
1088	Log(("ctrl.TLBCtrl.u7Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u7Reserved));
1089	Log(("ctrl.TLBCtrl.u24Reserved %x\n", pVMCB->ctrl.TLBCtrl.n.u24Reserved));
1090
1091	Log(("ctrl.IntCtrl.u8VTPR %x\n", pVMCB->ctrl.IntCtrl.n.u8VTPR));
1092	Log(("ctrl.IntCtrl.u1VIrqValid %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqValid));
1093	Log(("ctrl.IntCtrl.u7Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved));
1094	Log(("ctrl.IntCtrl.u4VIrqPriority %x\n", pVMCB->ctrl.IntCtrl.n.u4VIrqPriority));
1095	Log(("ctrl.IntCtrl.u1IgnoreTPR %x\n", pVMCB->ctrl.IntCtrl.n.u1IgnoreTPR));
1096	Log(("ctrl.IntCtrl.u3Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u3Reserved));
1097	Log(("ctrl.IntCtrl.u1VIrqMasking %x\n", pVMCB->ctrl.IntCtrl.n.u1VIrqMasking));
1098	Log(("ctrl.IntCtrl.u7Reserved2 %x\n", pVMCB->ctrl.IntCtrl.n.u7Reserved2));
1099	Log(("ctrl.IntCtrl.u8VIrqVector %x\n", pVMCB->ctrl.IntCtrl.n.u8VIrqVector));
1100	Log(("ctrl.IntCtrl.u24Reserved %x\n", pVMCB->ctrl.IntCtrl.n.u24Reserved));
1101
1102	Log(("ctrl.u64IntShadow %RX64\n", pVMCB->ctrl.u64IntShadow));
1103	Log(("ctrl.u64ExitCode %RX64\n", pVMCB->ctrl.u64ExitCode));
1104	Log(("ctrl.u64ExitInfo1 %RX64\n", pVMCB->ctrl.u64ExitInfo1));
1105	Log(("ctrl.u64ExitInfo2 %RX64\n", pVMCB->ctrl.u64ExitInfo2));
1106	Log(("ctrl.ExitIntInfo.u8Vector %x\n", pVMCB->ctrl.ExitIntInfo.n.u8Vector));
1107	Log(("ctrl.ExitIntInfo.u3Type %x\n", pVMCB->ctrl.ExitIntInfo.n.u3Type));
1108	Log(("ctrl.ExitIntInfo.u1ErrorCodeValid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
1109	Log(("ctrl.ExitIntInfo.u19Reserved %x\n", pVMCB->ctrl.ExitIntInfo.n.u19Reserved));
1110	Log(("ctrl.ExitIntInfo.u1Valid %x\n", pVMCB->ctrl.ExitIntInfo.n.u1Valid));
1111	Log(("ctrl.ExitIntInfo.u32ErrorCode %x\n", pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode));
1112	Log(("ctrl.NestedPaging %RX64\n", pVMCB->ctrl.NestedPaging.au64));
1113	Log(("ctrl.EventInject.u8Vector %x\n", pVMCB->ctrl.EventInject.n.u8Vector));
1114	Log(("ctrl.EventInject.u3Type %x\n", pVMCB->ctrl.EventInject.n.u3Type));
1115	Log(("ctrl.EventInject.u1ErrorCodeValid %x\n", pVMCB->ctrl.EventInject.n.u1ErrorCodeValid));
1116	Log(("ctrl.EventInject.u19Reserved %x\n", pVMCB->ctrl.EventInject.n.u19Reserved));
1117	Log(("ctrl.EventInject.u1Valid %x\n", pVMCB->ctrl.EventInject.n.u1Valid));
1118	Log(("ctrl.EventInject.u32ErrorCode %x\n", pVMCB->ctrl.EventInject.n.u32ErrorCode));
1119
1120	Log(("ctrl.u64NestedPagingCR3 %RX64\n", pVMCB->ctrl.u64NestedPagingCR3));
1121	Log(("ctrl.u64LBRVirt %RX64\n", pVMCB->ctrl.u64LBRVirt));
1122
1123	Log(("guest.CS.u16Sel %04X\n", pVMCB->guest.CS.u16Sel));
1124	Log(("guest.CS.u16Attr %04X\n", pVMCB->guest.CS.u16Attr));
1125	Log(("guest.CS.u32Limit %X\n", pVMCB->guest.CS.u32Limit));
1126	Log(("guest.CS.u64Base %RX64\n", pVMCB->guest.CS.u64Base));
1127	Log(("guest.DS.u16Sel %04X\n", pVMCB->guest.DS.u16Sel));
1128	Log(("guest.DS.u16Attr %04X\n", pVMCB->guest.DS.u16Attr));
1129	Log(("guest.DS.u32Limit %X\n", pVMCB->guest.DS.u32Limit));
1130	Log(("guest.DS.u64Base %RX64\n", pVMCB->guest.DS.u64Base));
1131	Log(("guest.ES.u16Sel %04X\n", pVMCB->guest.ES.u16Sel));
1132	Log(("guest.ES.u16Attr %04X\n", pVMCB->guest.ES.u16Attr));
1133	Log(("guest.ES.u32Limit %X\n", pVMCB->guest.ES.u32Limit));
1134	Log(("guest.ES.u64Base %RX64\n", pVMCB->guest.ES.u64Base));
1135	Log(("guest.FS.u16Sel %04X\n", pVMCB->guest.FS.u16Sel));
1136	Log(("guest.FS.u16Attr %04X\n", pVMCB->guest.FS.u16Attr));
1137	Log(("guest.FS.u32Limit %X\n", pVMCB->guest.FS.u32Limit));
1138	Log(("guest.FS.u64Base %RX64\n", pVMCB->guest.FS.u64Base));
1139	Log(("guest.GS.u16Sel %04X\n", pVMCB->guest.GS.u16Sel));
1140	Log(("guest.GS.u16Attr %04X\n", pVMCB->guest.GS.u16Attr));
1141	Log(("guest.GS.u32Limit %X\n", pVMCB->guest.GS.u32Limit));
1142	Log(("guest.GS.u64Base %RX64\n", pVMCB->guest.GS.u64Base));
1143
1144	Log(("guest.GDTR.u32Limit %X\n", pVMCB->guest.GDTR.u32Limit));
1145	Log(("guest.GDTR.u64Base %RX64\n", pVMCB->guest.GDTR.u64Base));
1146
1147	Log(("guest.LDTR.u16Sel %04X\n", pVMCB->guest.LDTR.u16Sel));
1148	Log(("guest.LDTR.u16Attr %04X\n", pVMCB->guest.LDTR.u16Attr));
1149	Log(("guest.LDTR.u32Limit %X\n", pVMCB->guest.LDTR.u32Limit));
1150	Log(("guest.LDTR.u64Base %RX64\n", pVMCB->guest.LDTR.u64Base));
1151
1152	Log(("guest.IDTR.u32Limit %X\n", pVMCB->guest.IDTR.u32Limit));
1153	Log(("guest.IDTR.u64Base %RX64\n", pVMCB->guest.IDTR.u64Base));
1154
1155	Log(("guest.TR.u16Sel %04X\n", pVMCB->guest.TR.u16Sel));
1156	Log(("guest.TR.u16Attr %04X\n", pVMCB->guest.TR.u16Attr));
1157	Log(("guest.TR.u32Limit %X\n", pVMCB->guest.TR.u32Limit));
1158	Log(("guest.TR.u64Base %RX64\n", pVMCB->guest.TR.u64Base));
1159
1160	Log(("guest.u8CPL %X\n", pVMCB->guest.u8CPL));
1161	Log(("guest.u64CR0 %RX64\n", pVMCB->guest.u64CR0));
1162	Log(("guest.u64CR2 %RX64\n", pVMCB->guest.u64CR2));
1163	Log(("guest.u64CR3 %RX64\n", pVMCB->guest.u64CR3));
1164	Log(("guest.u64CR4 %RX64\n", pVMCB->guest.u64CR4));
1165	Log(("guest.u64DR6 %RX64\n", pVMCB->guest.u64DR6));
1166	Log(("guest.u64DR7 %RX64\n", pVMCB->guest.u64DR7));
1167
1168	Log(("guest.u64RIP %RX64\n", pVMCB->guest.u64RIP));
1169	Log(("guest.u64RSP %RX64\n", pVMCB->guest.u64RSP));
1170	Log(("guest.u64RAX %RX64\n", pVMCB->guest.u64RAX));
1171	Log(("guest.u64RFlags %RX64\n", pVMCB->guest.u64RFlags));
1172
1173	Log(("guest.u64SysEnterCS %RX64\n", pVMCB->guest.u64SysEnterCS));
1174	Log(("guest.u64SysEnterEIP %RX64\n", pVMCB->guest.u64SysEnterEIP));
1175	Log(("guest.u64SysEnterESP %RX64\n", pVMCB->guest.u64SysEnterESP));
1176
1177	Log(("guest.u64EFER %RX64\n", pVMCB->guest.u64EFER));
1178	Log(("guest.u64STAR %RX64\n", pVMCB->guest.u64STAR));
1179	Log(("guest.u64LSTAR %RX64\n", pVMCB->guest.u64LSTAR));
1180	Log(("guest.u64CSTAR %RX64\n", pVMCB->guest.u64CSTAR));
1181	Log(("guest.u64SFMASK %RX64\n", pVMCB->guest.u64SFMASK));
1182	Log(("guest.u64KernelGSBase %RX64\n", pVMCB->guest.u64KernelGSBase));
1183	Log(("guest.u64GPAT %RX64\n", pVMCB->guest.u64GPAT));
1184	Log(("guest.u64DBGCTL %RX64\n", pVMCB->guest.u64DBGCTL));
1185	Log(("guest.u64BR_FROM %RX64\n", pVMCB->guest.u64BR_FROM));
1186	Log(("guest.u64BR_TO %RX64\n", pVMCB->guest.u64BR_TO));
1187	Log(("guest.u64LASTEXCPFROM %RX64\n", pVMCB->guest.u64LASTEXCPFROM));
1188	Log(("guest.u64LASTEXCPTO %RX64\n", pVMCB->guest.u64LASTEXCPTO));
1189
1190	#endif
1191	rc = VERR_SVM_UNABLE_TO_START_VM;
1192	goto end;
1193	}
1194
1195	/* Let's first sync back eip, esp, and eflags. */
1196	pCtx->rip = pVMCB->guest.u64RIP;
1197	pCtx->rsp = pVMCB->guest.u64RSP;
1198	pCtx->eflags.u32 = pVMCB->guest.u64RFlags;
1199	/* eax is saved/restore across the vmrun instruction */
1200	pCtx->rax = pVMCB->guest.u64RAX;
1201
1202	pCtx->msrKERNELGSBASE = pVMCB->guest.u64KernelGSBase; /* swapgs exchange value */
1203
1204	/* Can be updated behind our back in the nested paging case. */
1205	pCtx->cr2 = pVMCB->guest.u64CR2;
1206
1207	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1208	SVM_READ_SELREG(SS, ss);
1209	SVM_READ_SELREG(CS, cs);
1210	SVM_READ_SELREG(DS, ds);
1211	SVM_READ_SELREG(ES, es);
1212	SVM_READ_SELREG(FS, fs);
1213	SVM_READ_SELREG(GS, gs);
1214
1215	/*
1216	* System MSRs
1217	*/
1218	pCtx->SysEnter.cs = pVMCB->guest.u64SysEnterCS;
1219	pCtx->SysEnter.eip = pVMCB->guest.u64SysEnterEIP;
1220	pCtx->SysEnter.esp = pVMCB->guest.u64SysEnterESP;
1221
1222	/* Remaining guest CPU context: TR, IDTR, GDTR, LDTR; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1223	SVM_READ_SELREG(LDTR, ldtr);
1224	SVM_READ_SELREG(TR, tr);
1225
1226	pCtx->gdtr.cbGdt = pVMCB->guest.GDTR.u32Limit;
1227	pCtx->gdtr.pGdt = pVMCB->guest.GDTR.u64Base;
1228
1229	pCtx->idtr.cbIdt = pVMCB->guest.IDTR.u32Limit;
1230	pCtx->idtr.pIdt = pVMCB->guest.IDTR.u64Base;
1231
1232	/* Note: no reason to sync back the CRx and DRx registers. They can't be changed by the guest. */
1233	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1234	if ( pVM->hwaccm.s.fNestedPaging
1235	&& pCtx->cr3 != pVMCB->guest.u64CR3)
1236	{
1237	CPUMSetGuestCR3(pVM, pVMCB->guest.u64CR3);
1238	PGMUpdateCR3(pVM, pVMCB->guest.u64CR3);
1239	}
1240
1241	/* Note! NOW IT'S SAFE FOR LOGGING! */
1242
1243	/* Take care of instruction fusing (sti, mov ss) (see 15.20.5 Interrupt Shadows) */
1244	if (pVMCB->ctrl.u64IntShadow & SVM_INTERRUPT_SHADOW_ACTIVE)
1245	{
1246	Log(("uInterruptState %x rip=%RGv\n", pVMCB->ctrl.u64IntShadow, (RTGCPTR)pCtx->rip));
1247	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1248	}
1249	else
1250	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1251
1252	Log2(("exitCode = %x\n", exitCode));
1253
1254	/* Sync back DR6 as it could have been changed by hitting breakpoints. */
1255	pCtx->dr[6] = pVMCB->guest.u64DR6;
1256	/* DR7.GD can be cleared by debug exceptions, so sync it back as well. */
1257	pCtx->dr[7] = pVMCB->guest.u64DR7;
1258
1259	/* Check if an injected event was interrupted prematurely. */
1260	pVCpu->hwaccm.s.Event.intInfo = pVMCB->ctrl.ExitIntInfo.au64[0];
1261	if ( pVMCB->ctrl.ExitIntInfo.n.u1Valid
1262	&& pVMCB->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT /* we don't care about 'int xx' as the instruction will be restarted. */)
1263	{
1264	Log(("Pending inject %RX64 at %RGv exit=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitCode));
1265
1266	#ifdef LOG_ENABLED
1267	SVM_EVENT Event;
1268	Event.au64[0] = pVCpu->hwaccm.s.Event.intInfo;
1269
1270	if ( exitCode == SVM_EXIT_EXCEPTION_E
1271	&& Event.n.u8Vector == 0xE)
1272	{
1273	Log(("Double fault!\n"));
1274	}
1275	#endif
1276
1277	pVCpu->hwaccm.s.Event.fPending = true;
1278	/* Error code present? (redundant) */
1279	if (pVMCB->ctrl.ExitIntInfo.n.u1ErrorCodeValid)
1280	{
1281	pVCpu->hwaccm.s.Event.errCode = pVMCB->ctrl.ExitIntInfo.n.u32ErrorCode;
1282	}
1283	else
1284	pVCpu->hwaccm.s.Event.errCode = 0;
1285	}
1286	#ifdef VBOX_WITH_STATISTICS
1287	if (exitCode == SVM_EXIT_NPF)
1288	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
1289	else
1290	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitCode & MASK_EXITREASON_STAT]);
1291	#endif
1292
1293	if (fSyncTPR)
1294	{
1295	rc = PDMApicSetTPR(pVM, pVMCB->ctrl.IntCtrl.n.u8VTPR);
1296	AssertRC(rc);
1297	}
1298
1299	/* Deal with the reason of the VM-exit. */
1300	switch (exitCode)
1301	{
1302	case SVM_EXIT_EXCEPTION_0: case SVM_EXIT_EXCEPTION_1: case SVM_EXIT_EXCEPTION_2: case SVM_EXIT_EXCEPTION_3:
1303	case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5: case SVM_EXIT_EXCEPTION_6: case SVM_EXIT_EXCEPTION_7:
1304	case SVM_EXIT_EXCEPTION_8: case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_A: case SVM_EXIT_EXCEPTION_B:
1305	case SVM_EXIT_EXCEPTION_C: case SVM_EXIT_EXCEPTION_D: case SVM_EXIT_EXCEPTION_E: case SVM_EXIT_EXCEPTION_F:
1306	case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11: case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13:
1307	case SVM_EXIT_EXCEPTION_14: case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: case SVM_EXIT_EXCEPTION_17:
1308	case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_1A: case SVM_EXIT_EXCEPTION_1B:
1309	case SVM_EXIT_EXCEPTION_1C: case SVM_EXIT_EXCEPTION_1D: case SVM_EXIT_EXCEPTION_1E: case SVM_EXIT_EXCEPTION_1F:
1310	{
1311	/* Pending trap. */
1312	SVM_EVENT Event;
1313	uint32_t vector = exitCode - SVM_EXIT_EXCEPTION_0;
1314
1315	Log2(("Hardware/software interrupt %d\n", vector));
1316	switch (vector)
1317	{
1318	case X86_XCPT_DB:
1319	{
1320	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
1321
1322	/* Note that we don't support guest and host-initiated debugging at the same time. */
1323	Assert(DBGFIsStepping(pVM));
1324
1325	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), pCtx->dr[6]);
1326	if (rc == VINF_EM_RAW_GUEST_TRAP)
1327	{
1328	Log(("Trap %x (debug) at %016RX64\n", vector, pCtx->rip));
1329
1330	/* Reinject the exception. */
1331	Event.au64[0] = 0;
1332	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1333	Event.n.u1Valid = 1;
1334	Event.n.u8Vector = X86_XCPT_DB;
1335
1336	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1337
1338	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1339	goto ResumeExecution;
1340	}
1341	/* Return to ring 3 to deal with the debug exit code. */
1342	break;
1343	}
1344
1345	case X86_XCPT_NM:
1346	{
1347	Log(("#NM fault at %RGv\n", (RTGCPTR)pCtx->rip));
1348
1349	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1350	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1351	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
1352	if (rc == VINF_SUCCESS)
1353	{
1354	Assert(CPUMIsGuestFPUStateActive(pVCpu));
1355	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
1356
1357	/* Continue execution. */
1358	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1359	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1360
1361	goto ResumeExecution;
1362	}
1363
1364	Log(("Forward #NM fault to the guest\n"));
1365	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
1366
1367	Event.au64[0] = 0;
1368	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1369	Event.n.u1Valid = 1;
1370	Event.n.u8Vector = X86_XCPT_NM;
1371
1372	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1373	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1374	goto ResumeExecution;
1375	}
1376
1377	case X86_XCPT_PF: /* Page fault */
1378	{
1379	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1380	RTGCUINTPTR uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1381
1382	#ifdef DEBUG
1383	if (pVM->hwaccm.s.fNestedPaging)
1384	{ /* A genuine pagefault.
1385	* Forward the trap to the guest by injecting the exception and resuming execution.
1386	*/
1387	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode, (RTGCPTR)pCtx->rsp));
1388	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1389
1390	/* Now we must update CR2. */
1391	pCtx->cr2 = uFaultAddress;
1392
1393	Event.au64[0] = 0;
1394	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1395	Event.n.u1Valid = 1;
1396	Event.n.u8Vector = X86_XCPT_PF;
1397	Event.n.u1ErrorCodeValid = 1;
1398	Event.n.u32ErrorCode = errCode;
1399
1400	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1401
1402	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1403	goto ResumeExecution;
1404	}
1405	#endif
1406	Assert(!pVM->hwaccm.s.fNestedPaging);
1407
1408	Log2(("Page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1409	/* Exit qualification contains the linear address of the page fault. */
1410	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1411	TRPMSetErrorCode(pVM, errCode);
1412	TRPMSetFaultAddress(pVM, uFaultAddress);
1413
1414	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1415	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
1416	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1417	if (rc == VINF_SUCCESS)
1418	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1419	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1420	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1421
1422	TRPMResetTrap(pVM);
1423
1424	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1425	goto ResumeExecution;
1426	}
1427	else
1428	if (rc == VINF_EM_RAW_GUEST_TRAP)
1429	{ /* A genuine pagefault.
1430	* Forward the trap to the guest by injecting the exception and resuming execution.
1431	*/
1432	Log2(("Forward page fault to the guest\n"));
1433	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
1434	/* The error code might have been changed. */
1435	errCode = TRPMGetErrorCode(pVM);
1436
1437	TRPMResetTrap(pVM);
1438
1439	/* Now we must update CR2. */
1440	pCtx->cr2 = uFaultAddress;
1441
1442	Event.au64[0] = 0;
1443	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1444	Event.n.u1Valid = 1;
1445	Event.n.u8Vector = X86_XCPT_PF;
1446	Event.n.u1ErrorCodeValid = 1;
1447	Event.n.u32ErrorCode = errCode;
1448
1449	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1450
1451	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1452	goto ResumeExecution;
1453	}
1454	#ifdef VBOX_STRICT
1455	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1456	LogFlow(("PGMTrap0eHandler failed with %d\n", rc));
1457	#endif
1458	/* Need to go back to the recompiler to emulate the instruction. */
1459	TRPMResetTrap(pVM);
1460	break;
1461	}
1462
1463	case X86_XCPT_MF: /* Floating point exception. */
1464	{
1465	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
1466	if (!(pCtx->cr0 & X86_CR0_NE))
1467	{
1468	/* old style FPU error reporting needs some extra work. */
1469	/** @todo don't fall back to the recompiler, but do it manually. */
1470	rc = VINF_EM_RAW_EMULATE_INSTR;
1471	break;
1472	}
1473	Log(("Trap %x at %RGv\n", vector, (RTGCPTR)pCtx->rip));
1474
1475	Event.au64[0] = 0;
1476	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1477	Event.n.u1Valid = 1;
1478	Event.n.u8Vector = X86_XCPT_MF;
1479
1480	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1481
1482	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1483	goto ResumeExecution;
1484	}
1485
1486	#ifdef VBOX_STRICT
1487	case X86_XCPT_GP: /* General protection failure exception.*/
1488	case X86_XCPT_UD: /* Unknown opcode exception. */
1489	case X86_XCPT_DE: /* Divide error. */
1490	case X86_XCPT_SS: /* Stack segment exception. */
1491	case X86_XCPT_NP: /* Segment not present exception. */
1492	{
1493	Event.au64[0] = 0;
1494	Event.n.u3Type = SVM_EVENT_EXCEPTION;
1495	Event.n.u1Valid = 1;
1496	Event.n.u8Vector = vector;
1497
1498	switch(vector)
1499	{
1500	case X86_XCPT_GP:
1501	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
1502	Event.n.u1ErrorCodeValid = 1;
1503	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1504	break;
1505	case X86_XCPT_DE:
1506	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
1507	break;
1508	case X86_XCPT_UD:
1509	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
1510	break;
1511	case X86_XCPT_SS:
1512	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
1513	Event.n.u1ErrorCodeValid = 1;
1514	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1515	break;
1516	case X86_XCPT_NP:
1517	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
1518	Event.n.u1ErrorCodeValid = 1;
1519	Event.n.u32ErrorCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1520	break;
1521	}
1522	Log(("Trap %x at %RGv esi=%x\n", vector, (RTGCPTR)pCtx->rip, pCtx->esi));
1523	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1524
1525	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1526	goto ResumeExecution;
1527	}
1528	#endif
1529	default:
1530	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
1531	rc = VERR_EM_INTERNAL_ERROR;
1532	break;
1533
1534	} /* switch (vector) */
1535	break;
1536	}
1537
1538	case SVM_EXIT_NPF:
1539	{
1540	/* EXITINFO1 contains fault errorcode; EXITINFO2 contains the guest physical address causing the fault. */
1541	uint32_t errCode = pVMCB->ctrl.u64ExitInfo1; /* EXITINFO1 = error code */
1542	RTGCPHYS uFaultAddress = pVMCB->ctrl.u64ExitInfo2; /* EXITINFO2 = fault address */
1543	PGMMODE enmShwPagingMode;
1544
1545	Assert(pVM->hwaccm.s.fNestedPaging);
1546	Log(("Nested page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1547	/* Exit qualification contains the linear address of the page fault. */
1548	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1549	TRPMSetErrorCode(pVM, errCode);
1550	TRPMSetFaultAddress(pVM, uFaultAddress);
1551
1552	/* Handle the pagefault trap for the nested shadow table. */
1553	#if HC_ARCH_BITS == 32
1554	if (CPUMIsGuestInLongModeEx(pCtx))
1555	enmShwPagingMode = PGMMODE_AMD64_NX;
1556	else
1557	#endif
1558	enmShwPagingMode = PGMGetHostMode(pVM);
1559
1560	rc = PGMR0Trap0eHandlerNestedPaging(pVM, enmShwPagingMode, errCode, CPUMCTX2CORE(pCtx), uFaultAddress);
1561	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
1562	if (rc == VINF_SUCCESS)
1563	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1564	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, uFaultAddress, errCode));
1565	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
1566
1567	TRPMResetTrap(pVM);
1568
1569	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1570	goto ResumeExecution;
1571	}
1572
1573	#ifdef VBOX_STRICT
1574	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1575	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
1576	#endif
1577	/* Need to go back to the recompiler to emulate the instruction. */
1578	TRPMResetTrap(pVM);
1579	break;
1580	}
1581
1582	case SVM_EXIT_VINTR:
1583	/* A virtual interrupt is about to be delivered, which means IF=1. */
1584	Log(("SVM_EXIT_VINTR IF=%d\n", pCtx->eflags.Bits.u1IF));
1585	pVMCB->ctrl.IntCtrl.n.u1VIrqValid = 0;
1586	pVMCB->ctrl.IntCtrl.n.u8VIrqVector = 0;
1587	goto ResumeExecution;
1588
1589	case SVM_EXIT_FERR_FREEZE:
1590	case SVM_EXIT_INTR:
1591	case SVM_EXIT_NMI:
1592	case SVM_EXIT_SMI:
1593	case SVM_EXIT_INIT:
1594	/* External interrupt; leave to allow it to be dispatched again. */
1595	rc = VINF_EM_RAW_INTERRUPT;
1596	break;
1597
1598	case SVM_EXIT_WBINVD:
1599	case SVM_EXIT_INVD: /* Guest software attempted to execute INVD. */
1600	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
1601	/* Skip instruction and continue directly. */
1602	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1603	/* Continue execution.*/
1604	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1605	goto ResumeExecution;
1606
1607	case SVM_EXIT_CPUID: /* Guest software attempted to execute CPUID. */
1608	{
1609	Log2(("SVM: Cpuid at %RGv for %x\n", (RTGCPTR)pCtx->rip, pCtx->eax));
1610	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
1611	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
1612	if (rc == VINF_SUCCESS)
1613	{
1614	/* Update EIP and continue execution. */
1615	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1616	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1617	goto ResumeExecution;
1618	}
1619	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
1620	rc = VINF_EM_RAW_EMULATE_INSTR;
1621	break;
1622	}
1623
1624	case SVM_EXIT_RDTSC: /* Guest software attempted to execute RDTSC. */
1625	{
1626	Log2(("SVM: Rdtsc\n"));
1627	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1628	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
1629	if (rc == VINF_SUCCESS)
1630	{
1631	/* Update EIP and continue execution. */
1632	pCtx->rip += 2; /* Note! hardcoded opcode size! */
1633	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1634	goto ResumeExecution;
1635	}
1636	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
1637	rc = VINF_EM_RAW_EMULATE_INSTR;
1638	break;
1639	}
1640
1641	case SVM_EXIT_RDTSCP: /* Guest software attempted to execute RDTSCP. */
1642	{
1643	Log2(("SVM: Rdtscp\n"));
1644	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
1645	rc = EMInterpretRdtscp(pVM, pCtx);
1646	if (rc == VINF_SUCCESS)
1647	{
1648	/* Update EIP and continue execution. */
1649	pCtx->rip += 3; /* Note! hardcoded opcode size! */
1650	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1651	goto ResumeExecution;
1652	}
1653	AssertMsgFailed(("EMU: rdtscp failed with %Rrc\n", rc));
1654	rc = VINF_EM_RAW_EMULATE_INSTR;
1655	break;
1656	}
1657
1658	case SVM_EXIT_INVLPG: /* Guest software attempted to execute INVPG. */
1659	{
1660	Log2(("SVM: invlpg\n"));
1661	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
1662
1663	Assert(!pVM->hwaccm.s.fNestedPaging);
1664
1665	/* Truly a pita. Why can't SVM give the same information as VT-x? */
1666	rc = SVMR0InterpretInvpg(pVM, CPUMCTX2CORE(pCtx), pVMCB->ctrl.TLBCtrl.n.u32ASID);
1667	if (rc == VINF_SUCCESS)
1668	{
1669	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageInvlpg);
1670	goto ResumeExecution; /* eip already updated */
1671	}
1672	break;
1673	}
1674
1675	case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
1676	case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
1677	case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
1678	case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
1679	{
1680	uint32_t cbSize;
1681
1682	Log2(("SVM: %RGv mov cr%d, \n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_CR0));
1683	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[exitCode - SVM_EXIT_WRITE_CR0]);
1684	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1685
1686	switch (exitCode - SVM_EXIT_WRITE_CR0)
1687	{
1688	case 0:
1689	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1690	break;
1691	case 2:
1692	break;
1693	case 3:
1694	Assert(!pVM->hwaccm.s.fNestedPaging);
1695	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
1696	break;
1697	case 4:
1698	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
1699	break;
1700	case 8:
1701	break;
1702	default:
1703	AssertFailed();
1704	}
1705	/* Check if a sync operation is pending. */
1706	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
1707	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
1708	{
1709	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
1710	AssertRC(rc);
1711
1712	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBCRxChange);
1713
1714	/* Must be set by PGMSyncCR3 */
1715	Assert(PGMGetGuestMode(pVM) <= PGMMODE_PROTECTED \|\| pVCpu->hwaccm.s.fForceTLBFlush);
1716	}
1717	if (rc == VINF_SUCCESS)
1718	{
1719	/* EIP has been updated already. */
1720
1721	/* Only resume if successful. */
1722	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1723	goto ResumeExecution;
1724	}
1725	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1726	break;
1727	}
1728
1729	case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
1730	case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
1731	case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
1732	case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
1733	{
1734	uint32_t cbSize;
1735
1736	Log2(("SVM: %RGv mov x, cr%d\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_CR0));
1737	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[exitCode - SVM_EXIT_WRITE_CR0]);
1738	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1739	if (rc == VINF_SUCCESS)
1740	{
1741	/* EIP has been updated already. */
1742
1743	/* Only resume if successful. */
1744	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1745	goto ResumeExecution;
1746	}
1747	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1748	break;
1749	}
1750
1751	case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
1752	case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
1753	case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
1754	case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
1755	{
1756	uint32_t cbSize;
1757
1758	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_WRITE_DR0));
1759	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
1760
1761	if (!DBGFIsStepping(pVM))
1762	{
1763	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1764
1765	/* Disable drx move intercepts. */
1766	pVMCB->ctrl.u16InterceptRdDRx = 0;
1767	pVMCB->ctrl.u16InterceptWrDRx = 0;
1768
1769	/* Save the host and load the guest debug state. */
1770	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1771	AssertRC(rc);
1772
1773	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1774	goto ResumeExecution;
1775	}
1776
1777	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1778	if (rc == VINF_SUCCESS)
1779	{
1780	/* EIP has been updated already. */
1781	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
1782
1783	/* Only resume if successful. */
1784	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1785	goto ResumeExecution;
1786	}
1787	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1788	break;
1789	}
1790
1791	case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
1792	case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
1793	case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
1794	case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
1795	{
1796	uint32_t cbSize;
1797
1798	Log2(("SVM: %RGv mov dr%d, x\n", (RTGCPTR)pCtx->rip, exitCode - SVM_EXIT_READ_DR0));
1799	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
1800
1801	if (!DBGFIsStepping(pVM))
1802	{
1803	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
1804
1805	/* Disable drx move intercepts. */
1806	pVMCB->ctrl.u16InterceptRdDRx = 0;
1807	pVMCB->ctrl.u16InterceptWrDRx = 0;
1808
1809	/* Save the host and load the guest debug state. */
1810	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, false /* exclude DR6 */);
1811	AssertRC(rc);
1812
1813	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1814	goto ResumeExecution;
1815	}
1816
1817	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
1818	if (rc == VINF_SUCCESS)
1819	{
1820	/* EIP has been updated already. */
1821
1822	/* Only resume if successful. */
1823	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1824	goto ResumeExecution;
1825	}
1826	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
1827	break;
1828	}
1829
1830	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
1831	case SVM_EXIT_IOIO: /* I/O instruction. */
1832	{
1833	SVM_IOIO_EXIT IoExitInfo;
1834	uint32_t uIOSize, uAndVal;
1835
1836	IoExitInfo.au32[0] = pVMCB->ctrl.u64ExitInfo1;
1837
1838	/** @todo could use a lookup table here */
1839	if (IoExitInfo.n.u1OP8)
1840	{
1841	uIOSize = 1;
1842	uAndVal = 0xff;
1843	}
1844	else
1845	if (IoExitInfo.n.u1OP16)
1846	{
1847	uIOSize = 2;
1848	uAndVal = 0xffff;
1849	}
1850	else
1851	if (IoExitInfo.n.u1OP32)
1852	{
1853	uIOSize = 4;
1854	uAndVal = 0xffffffff;
1855	}
1856	else
1857	{
1858	AssertFailed(); /* should be fatal. */
1859	rc = VINF_EM_RAW_EMULATE_INSTR;
1860	break;
1861	}
1862
1863	if (IoExitInfo.n.u1STR)
1864	{
1865	/* ins/outs */
1866	DISCPUSTATE Cpu;
1867
1868	/* Disassemble manually to deal with segment prefixes. */
1869	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
1870	if (rc == VINF_SUCCESS)
1871	{
1872	if (IoExitInfo.n.u1Type == 0)
1873	{
1874	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1875	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
1876	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, Cpu.prefix, uIOSize);
1877	}
1878	else
1879	{
1880	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1881	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
1882	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, Cpu.prefix, uIOSize);
1883	}
1884	}
1885	else
1886	rc = VINF_EM_RAW_EMULATE_INSTR;
1887	}
1888	else
1889	{
1890	/* normal in/out */
1891	Assert(!IoExitInfo.n.u1REP);
1892
1893	if (IoExitInfo.n.u1Type == 0)
1894	{
1895	Log2(("IOMIOPortWrite %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize));
1896	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
1897	rc = IOMIOPortWrite(pVM, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, uIOSize);
1898	}
1899	else
1900	{
1901	uint32_t u32Val = 0;
1902
1903	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
1904	rc = IOMIOPortRead(pVM, IoExitInfo.n.u16Port, &u32Val, uIOSize);
1905	if (IOM_SUCCESS(rc))
1906	{
1907	/* Write back to the EAX register. */
1908	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
1909	Log2(("IOMIOPortRead %RGv %x %x size=%d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, u32Val & uAndVal, uIOSize));
1910	}
1911	}
1912	}
1913	/*
1914	* Handled the I/O return codes.
1915	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
1916	*/
1917	if (IOM_SUCCESS(rc))
1918	{
1919	/* Update EIP and continue execution. */
1920	pCtx->rip = pVMCB->ctrl.u64ExitInfo2; /* RIP/EIP of the next instruction is saved in EXITINFO2. */
1921	if (RT_LIKELY(rc == VINF_SUCCESS))
1922	{
1923	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
1924	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
1925	{
1926	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
1927	for (unsigned i=0;i<4;i++)
1928	{
1929	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
1930
1931	if ( (IoExitInfo.n.u16Port >= pCtx->dr[i] && IoExitInfo.n.u16Port < pCtx->dr[i] + uBPLen)
1932	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
1933	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
1934	{
1935	SVM_EVENT Event;
1936
1937	Assert(CPUMIsGuestDebugStateActive(pVM));
1938
1939	/* Clear all breakpoint status flags and set the one we just hit. */
1940	pCtx->dr[6] &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
1941	pCtx->dr[6] \|= (uint64_t)RT_BIT(i);
1942
1943	/* Note: AMD64 Architecture Programmer's Manual 13.1:
1944	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
1945	* the contents have been read.
1946	*/
1947	pVMCB->guest.u64DR6 = pCtx->dr[6];
1948
1949	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
1950	pCtx->dr[7] &= ~X86_DR7_GD;
1951
1952	/* Paranoia. */
1953	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1954	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1955	pCtx->dr[7] \|= 0x400; /* must be one */
1956
1957	pVMCB->guest.u64DR7 = pCtx->dr[7];
1958
1959	/* Inject the exception. */
1960	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
1961
1962	Event.au64[0] = 0;
1963	Event.n.u3Type = SVM_EVENT_EXCEPTION; /* trap or fault */
1964	Event.n.u1Valid = 1;
1965	Event.n.u8Vector = X86_XCPT_DB;
1966
1967	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
1968
1969	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1970	goto ResumeExecution;
1971	}
1972	}
1973	}
1974
1975	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
1976	goto ResumeExecution;
1977	}
1978	Log2(("EM status from IO at %RGv %x size %d: %Rrc\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize, rc));
1979	break;
1980	}
1981
1982	#ifdef VBOX_STRICT
1983	if (rc == VINF_IOM_HC_IOPORT_READ)
1984	Assert(IoExitInfo.n.u1Type != 0);
1985	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
1986	Assert(IoExitInfo.n.u1Type == 0);
1987	else
1988	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
1989	#endif
1990	Log2(("Failed IO at %RGv %x size %d\n", (RTGCPTR)pCtx->rip, IoExitInfo.n.u16Port, uIOSize));
1991	break;
1992	}
1993
1994	case SVM_EXIT_HLT:
1995	/** Check if external interrupts are pending; if so, don't switch back. */
1996	pCtx->rip++; /* skip hlt */
1997	if ( pCtx->eflags.Bits.u1IF
1998	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
1999	goto ResumeExecution;
2000
2001	rc = VINF_EM_HALT;
2002	break;
2003
2004	case SVM_EXIT_RSM:
2005	case SVM_EXIT_INVLPGA:
2006	case SVM_EXIT_VMRUN:
2007	case SVM_EXIT_VMMCALL:
2008	case SVM_EXIT_VMLOAD:
2009	case SVM_EXIT_VMSAVE:
2010	case SVM_EXIT_STGI:
2011	case SVM_EXIT_CLGI:
2012	case SVM_EXIT_SKINIT:
2013	{
2014	/* Unsupported instructions. */
2015	SVM_EVENT Event;
2016
2017	Event.au64[0] = 0;
2018	Event.n.u3Type = SVM_EVENT_EXCEPTION;
2019	Event.n.u1Valid = 1;
2020	Event.n.u8Vector = X86_XCPT_UD;
2021
2022	Log(("Forced #UD trap at %RGv\n", (RTGCPTR)pCtx->rip));
2023	SVMR0InjectEvent(pVM, pVMCB, pCtx, &Event);
2024
2025	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2026	goto ResumeExecution;
2027	}
2028
2029	/* Emulate in ring 3. */
2030	case SVM_EXIT_MSR:
2031	{
2032	uint32_t cbSize;
2033
2034	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2035	Log(("SVM: %s\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr"));
2036	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2037	if (rc == VINF_SUCCESS)
2038	{
2039	/* EIP has been updated already. */
2040
2041	/* Only resume if successful. */
2042	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2043	goto ResumeExecution;
2044	}
2045	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (pVMCB->ctrl.u64ExitInfo1 == 0) ? "rdmsr" : "wrmsr", rc));
2046	break;
2047	}
2048
2049	case SVM_EXIT_MONITOR:
2050	case SVM_EXIT_RDPMC:
2051	case SVM_EXIT_PAUSE:
2052	case SVM_EXIT_MWAIT_UNCOND:
2053	case SVM_EXIT_MWAIT_ARMED:
2054	case SVM_EXIT_TASK_SWITCH: /* can change CR3; emulate */
2055	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2056	break;
2057
2058	case SVM_EXIT_SHUTDOWN:
2059	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2060	break;
2061
2062	case SVM_EXIT_IDTR_READ:
2063	case SVM_EXIT_GDTR_READ:
2064	case SVM_EXIT_LDTR_READ:
2065	case SVM_EXIT_TR_READ:
2066	case SVM_EXIT_IDTR_WRITE:
2067	case SVM_EXIT_GDTR_WRITE:
2068	case SVM_EXIT_LDTR_WRITE:
2069	case SVM_EXIT_TR_WRITE:
2070	case SVM_EXIT_CR0_SEL_WRITE:
2071	default:
2072	/* Unexpected exit codes. */
2073	rc = VERR_EM_INTERNAL_ERROR;
2074	AssertMsgFailed(("Unexpected exit code %x\n", exitCode)); /* Can't happen. */
2075	break;
2076	}
2077
2078	end:
2079
2080	/* Signal changes for the recompiler. */
2081	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
2082
2083	/* If we executed vmrun and an external irq was pending, then we don't have to do a full sync the next time. */
2084	if (exitCode == SVM_EXIT_INTR)
2085	{
2086	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
2087	/* On the next entry we'll only sync the host context. */
2088	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
2089	}
2090	else
2091	{
2092	/* On the next entry we'll sync everything. */
2093	/** @todo we can do better than this */
2094	/* Not in the VINF_PGM_CHANGE_MODE though! */
2095	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2096	}
2097
2098	/* translate into a less severe return code */
2099	if (rc == VERR_EM_INTERPRETER)
2100	rc = VINF_EM_RAW_EMULATE_INSTR;
2101
2102	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
2103	return rc;
2104	}
2105
2106	/**
2107	* Enters the AMD-V session
2108	*
2109	* @returns VBox status code.
2110	* @param pVM The VM to operate on.
2111	* @param pVCpu The VM CPU to operate on.
2112	* @param pCpu CPU info struct
2113	*/
2114	VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
2115	{
2116	Assert(pVM->hwaccm.s.svm.fSupported);
2117
2118	LogFlow(("SVMR0Enter cpu%d last=%d asid=%d\n", pCpu->idCpu, pVCpu->hwaccm.s.idLastCpu, pVCpu->hwaccm.s.uCurrentASID));
2119	pVCpu->hwaccm.s.fResumeVM = false;
2120
2121	/* Force to reload LDTR, so we'll execute VMLoad to load additional guest state. */
2122	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_LDTR;
2123
2124	return VINF_SUCCESS;
2125	}
2126
2127
2128	/**
2129	* Leaves the AMD-V session
2130	*
2131	* @returns VBox status code.
2132	* @param pVM The VM to operate on.
2133	* @param pVCpu The VM CPU to operate on.
2134	* @param pCtx CPU context
2135	*/
2136	VMMR0DECL(int) SVMR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2137	{
2138	SVM_VMCB pVMCB = (SVM_VMCB )pVCpu->hwaccm.s.svm.pVMCB;
2139
2140	Assert(pVM->hwaccm.s.svm.fSupported);
2141
2142	/* Save the guest debug state if necessary. */
2143	if (CPUMIsGuestDebugStateActive(pVM))
2144	{
2145	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, false /* skip DR6 */);
2146
2147	/* Intercept all DRx reads and writes again. Changed later on. */
2148	pVMCB->ctrl.u16InterceptRdDRx = 0xFFFF;
2149	pVMCB->ctrl.u16InterceptWrDRx = 0xFFFF;
2150
2151	/* Resync the debug registers the next time. */
2152	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2153	}
2154	else
2155	Assert(pVMCB->ctrl.u16InterceptRdDRx == 0xFFFF && pVMCB->ctrl.u16InterceptWrDRx == 0xFFFF);
2156
2157	return VINF_SUCCESS;
2158	}
2159
2160
2161	static int svmR0InterpretInvlPg(PVM pVM, PDISCPUSTATE pCpu, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2162	{
2163	OP_PARAMVAL param1;
2164	RTGCPTR addr;
2165
2166	int rc = DISQueryParamVal(pRegFrame, pCpu, &pCpu->param1, &param1, PARAM_SOURCE);
2167	if(RT_FAILURE(rc))
2168	return VERR_EM_INTERPRETER;
2169
2170	switch(param1.type)
2171	{
2172	case PARMTYPE_IMMEDIATE:
2173	case PARMTYPE_ADDRESS:
2174	if(!(param1.flags & (PARAM_VAL32\|PARAM_VAL64)))
2175	return VERR_EM_INTERPRETER;
2176	addr = param1.val.val64;
2177	break;
2178
2179	default:
2180	return VERR_EM_INTERPRETER;
2181	}
2182
2183	/** @todo is addr always a flat linear address or ds based
2184	* (in absence of segment override prefixes)????
2185	*/
2186	rc = PGMInvalidatePage(pVM, addr);
2187	if (RT_SUCCESS(rc))
2188	{
2189	/* Manually invalidate the page for the VM's TLB. */
2190	Log(("SVMR0InvlpgA %RGv ASID=%d\n", addr, uASID));
2191	SVMR0InvlpgA(addr, uASID);
2192	return VINF_SUCCESS;
2193	}
2194	Assert(rc == VERR_REM_FLUSHED_PAGES_OVERFLOW);
2195	return rc;
2196	}
2197
2198	/**
2199	* Interprets INVLPG
2200	*
2201	* @returns VBox status code.
2202	* @retval VINF_* Scheduling instructions.
2203	* @retval VERR_EM_INTERPRETER Something we can't cope with.
2204	* @retval VERR_* Fatal errors.
2205	*
2206	* @param pVM The VM handle.
2207	* @param pRegFrame The register frame.
2208	* @param ASID Tagged TLB id for the guest
2209	*
2210	* Updates the EIP if an instruction was executed successfully.
2211	*/
2212	static int SVMR0InterpretInvpg(PVM pVM, PCPUMCTXCORE pRegFrame, uint32_t uASID)
2213	{
2214	/*
2215	* Only allow 32 & 64 bits code.
2216	*/
2217	DISCPUMODE enmMode = SELMGetCpuModeFromSelector(pVM, pRegFrame->eflags, pRegFrame->cs, &pRegFrame->csHid);
2218	if (enmMode != CPUMODE_16BIT)
2219	{
2220	RTGCPTR pbCode;
2221	int rc = SELMValidateAndConvertCSAddr(pVM, pRegFrame->eflags, pRegFrame->ss, pRegFrame->cs, &pRegFrame->csHid, (RTGCPTR)pRegFrame->rip, &pbCode);
2222	if (RT_SUCCESS(rc))
2223	{
2224	uint32_t cbOp;
2225	DISCPUSTATE Cpu;
2226
2227	Cpu.mode = enmMode;
2228	rc = EMInterpretDisasOneEx(pVM, pbCode, pRegFrame, &Cpu, &cbOp);
2229	Assert(RT_FAILURE(rc) \|\| Cpu.pCurInstr->opcode == OP_INVLPG);
2230	if (RT_SUCCESS(rc) && Cpu.pCurInstr->opcode == OP_INVLPG)
2231	{
2232	Assert(cbOp == Cpu.opsize);
2233	rc = svmR0InterpretInvlPg(pVM, &Cpu, pRegFrame, uASID);
2234	if (RT_SUCCESS(rc))
2235	{
2236	pRegFrame->rip += cbOp; /* Move on to the next instruction. */
2237	}
2238	return rc;
2239	}
2240	}
2241	}
2242	return VERR_EM_INTERPRETER;
2243	}
2244
2245
2246	/**
2247	* Invalidates a guest page
2248	*
2249	* @returns VBox status code.
2250	* @param pVM The VM to operate on.
2251	* @param pVCpu The VM CPU to operate on.
2252	* @param GCVirt Page to invalidate
2253	*/
2254	VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
2255	{
2256	bool fFlushPending = pVM->hwaccm.s.svm.fAlwaysFlushTLB \| pVCpu->hwaccm.s.fForceTLBFlush;
2257
2258	/* Skip it if a TLB flush is already pending. */
2259	if (!fFlushPending)
2260	{
2261	SVM_VMCB *pVMCB;
2262
2263	Log2(("SVMR0InvalidatePage %RGv\n", GCVirt));
2264	AssertReturn(pVM, VERR_INVALID_PARAMETER);
2265	Assert(pVM->hwaccm.s.svm.fSupported);
2266
2267	/* @todo SMP */
2268	pVMCB = (SVM_VMCB *)pVM->aCpus[0].hwaccm.s.svm.pVMCB;
2269	AssertMsgReturn(pVMCB, ("Invalid pVMCB\n"), VERR_EM_INTERNAL_ERROR);
2270
2271	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushPageManual);
2272	#if HC_ARCH_BITS == 32
2273	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invlpga takes only 32 bits addresses. */
2274	if (CPUMIsGuestInLongMode(pVM))
2275	pVCpu->hwaccm.s.fForceTLBFlush = true;
2276	else
2277	#endif
2278	SVMR0InvlpgA(GCVirt, pVMCB->ctrl.TLBCtrl.n.u32ASID);
2279	}
2280	return VINF_SUCCESS;
2281	}
2282
2283
2284	/**
2285	* Invalidates a guest page by physical address
2286	*
2287	* @returns VBox status code.
2288	* @param pVM The VM to operate on.
2289	* @param pVCpu The VM CPU to operate on.
2290	* @param GCPhys Page to invalidate
2291	*/
2292	VMMR0DECL(int) SVMR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
2293	{
2294	Assert(pVM->hwaccm.s.fNestedPaging);
2295	/* invlpga only invalidates TLB entries for guest virtual addresses; we have no choice but to force a TLB flush here. */
2296	pVCpu->hwaccm.s.fForceTLBFlush = true;
2297	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBInvlpga);
2298	return VINF_SUCCESS;
2299	}
2300
2301	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2302	/**
2303	* Prepares for and executes VMRUN (64 bits guests from a 32 bits hosts).
2304	*
2305	* @returns VBox status code.
2306	* @param pVMCBHostPhys Physical address of host VMCB.
2307	* @param pVMCBPhys Physical address of the VMCB.
2308	* @param pCtx Guest context.
2309	* @param pVM The VM to operate on.
2310	* @param pVCpu The VMCPU to operate on.
2311	*/
2312	DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS pVMCBHostPhys, RTHCPHYS pVMCBPhys, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
2313	{
2314	uint32_t aParam[4];
2315
2316	aParam[0] = (uint32_t)(pVMCBHostPhys); /* Param 1: pVMCBHostPhys - Lo. */
2317	aParam[1] = (uint32_t)(pVMCBHostPhys >> 32); /* Param 1: pVMCBHostPhys - Hi. */
2318	aParam[2] = (uint32_t)(pVMCBPhys); /* Param 2: pVMCBPhys - Lo. */
2319	aParam[3] = (uint32_t)(pVMCBPhys >> 32); /* Param 2: pVMCBPhys - Hi. */
2320
2321	return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnSVMGCVMRun64, 4, &aParam[0]);
2322	}
2323
2324	/**
2325	* Executes the specified handler in 64 mode
2326	*
2327	* @returns VBox status code.
2328	* @param pVM The VM to operate on.
2329	* @param pVCpu The VMCPU to operate on.
2330	* @param pCtx Guest context
2331	* @param pfnHandler RC handler
2332	* @param cbParam Number of parameters
2333	* @param paParam Array of 32 bits parameters
2334	*/
2335	VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
2336	{
2337	int rc;
2338	RTHCUINTREG uFlags;
2339
2340	/* @todo This code is not guest SMP safe (hyper context) */
2341	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
2342	Assert(pfnHandler);
2343
2344	uFlags = ASMIntDisableFlags();
2345
2346	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
2347	CPUMSetHyperEIP(pVM, pfnHandler);
2348	for (int i=(int)cbParam-1;i>=0;i--)
2349	CPUMPushHyper(pVM, paParam[i]);
2350
2351	/* Call switcher. */
2352	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
2353
2354	ASMSetFlags(uFlags);
2355	return rc;
2356	}
2357
2358	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWSVMR0.cpp@ 15733

Download in other formats: